diff --git a/.cache/config/.gitkeep b/.cache/config/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.cache/keras/.gitkeep b/.cache/keras/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.cache/nv/.gitkeep b/.cache/nv/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.cache/triton/.gitkeep b/.cache/triton/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.cache/user/.gitkeep b/.cache/user/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000000000000000000000000000000000000..8a134bbc7ef3757a0b822bb6937c627f4d314585
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,7 @@
+.cache/
+cudnn_windows/
+bitsandbytes_windows/
+bitsandbytes_windows_deprecated/
+dataset/
+__pycache__/
+venv/
diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..e0f979fb8faeb40adf3c67b70fbfce90c9544070 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,35 +1,4 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.sh text eol=lf
+*.ps1 text eol=crlf
+*.bat text eol=crlf
+*.cmd text eol=crlfbitsandbytes_windows/libbitsandbytes_cuda116.dll filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/workflows/typos.yaml b/.github/workflows/typos.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a7c2a8f0646a6421b114be99cc84e4ec8becfeab
--- /dev/null
+++ b/.github/workflows/typos.yaml
@@ -0,0 +1,21 @@
+---
+# yamllint disable rule:line-length
+name: Typos
+
+on:  # yamllint disable-line rule:truthy
+  push:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: typos-action
+        uses: crate-ci/typos@v1.13.10
\ No newline at end of file
diff --git a/.github/workflows/update_space.yml b/.github/workflows/update_space.yml
new file mode 100644
index 0000000000000000000000000000000000000000..67dbc84e4e59320a7c98b94460eb976e5cd2984f
--- /dev/null
+++ b/.github/workflows/update_space.yml
@@ -0,0 +1,28 @@
+name: Run Python script
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install Gradio
+      run: python -m pip install gradio
+
+    - name: Log in to Hugging Face
+      run: python -c 'import huggingface_hub; huggingface_hub.login(token="${{ secrets.hf_token }}")'
+
+    - name: Deploy to Spaces
+      run: gradio deploy
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..99734cc19c9f9594341f2812a4428f7a6ab94e57
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,47 @@
+# Python
+venv
+__pycache__
+*.egg-info
+build
+wd14_tagger_model
+
+# IDE and Editor specific
+.vscode
+
+# CUDNN for Windows
+cudnn_windows
+
+# Cache and temporary files
+.cache
+.DS_Store
+
+# Scripts and executables
+locon
+gui-user.bat
+gui-user.ps1
+
+# Version control
+SmilingWolf
+wandb
+
+# Setup and logs
+setup.log
+logs
+
+# Miscellaneous
+uninstall.txt
+
+# Test files
+test/output
+test/log*
+test/*.json
+test/ft
+
+# Temporary requirements
+requirements_tmp_for_setup.txt
+
+# Version specific
+0.13.3
+
+*.npz
+presets/*/user_presets/*
\ No newline at end of file
diff --git a/.release b/.release
new file mode 100644
index 0000000000000000000000000000000000000000..3f77139581f14530323051be74489683f75ff1ff
--- /dev/null
+++ b/.release
@@ -0,0 +1 @@
+v21.8.2
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000000000000000000000000000000000000..c08f3e9a0281e80e8e8949b4c1ba75c3f583cb8e
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,4 @@
+{
+    "python.linting.enabled": true,
+    "python.formatting.provider": "yapf"
+}
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..479783d04f4a28b5a10a77ee4e8f2c4aebef5705
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,52 @@
+FROM nvcr.io/nvidia/pytorch:23.04-py3 as base
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/London
+
+RUN apt update && apt-get install -y software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa && \
+    apt update && \
+    apt-get install -y git curl libgl1 libglib2.0-0 libgoogle-perftools-dev \
+    python3.10-dev python3.10-tk python3-html5lib python3-apt python3-pip python3.10-distutils && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set python 3.10 and cuda 11.8 as default
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 3 && \
+    update-alternatives --set python3 /usr/bin/python3.10 && \
+    update-alternatives --set cuda /usr/local/cuda-11.8
+
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3
+
+WORKDIR /app
+RUN python3 -m pip install wheel
+
+# Todo: Install torch 2.1.0 for cu121 support (only available as nightly as of writing)
+## RUN python3 -m pip install --pre torch ninja setuptools --extra-index-url https://download.pytorch.org/whl/nightly/cu121
+
+# Todo: Install xformers nightly for Torch 2.1.0 support
+## RUN python3 -m pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers
+
+# Install requirements
+COPY ./requirements.txt ./requirements_linux_docker.txt ./
+COPY ./setup/docker_setup.py ./setup.py
+RUN python3 -m pip install -r ./requirements_linux_docker.txt
+RUN python3 -m pip install -r ./requirements.txt
+
+# Replace pillow with pillow-simd
+RUN python3 -m pip uninstall -y pillow && \
+    CC="cc -mavx2" python3 -m pip install -U --force-reinstall pillow-simd
+
+# Fix missing libnvinfer7
+USER root
+RUN ln -s /usr/lib/x86_64-linux-gnu/libnvinfer.so /usr/lib/x86_64-linux-gnu/libnvinfer.so.7 && \
+    ln -s /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so.7
+
+RUN useradd -m -s /bin/bash appuser && \
+    chown -R appuser: /app
+USER appuser
+COPY --chown=appuser . .
+
+STOPSIGNAL SIGINT
+ENV LD_PRELOAD=libtcmalloc.so
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ENV PATH="$PATH:/home/appuser/.local/bin"
+CMD python3 "./kohya_gui.py" ${CLI_ARGS} --listen 0.0.0.0 --server_port 7860
diff --git a/LICENSE.md b/LICENSE.md
new file mode 100644
index 0000000000000000000000000000000000000000..56765e795c61de9b5941ba9d9c6379f0b7922203
--- /dev/null
+++ b/LICENSE.md
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [2022] [kohya-ss]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/README-ja.md b/README-ja.md
new file mode 100644
index 0000000000000000000000000000000000000000..6f0e574e20ec144a3e4fd843c2841a83fbd2c09b
--- /dev/null
+++ b/README-ja.md
@@ -0,0 +1,157 @@
+## リポジトリについて
+Stable Diffusionの学習、画像生成、その他のスクリプトを入れたリポジトリです。
+
+[README in English](./README.md) ←更新情報はこちらにあります
+
+GUIやPowerShellスクリプトなど、より使いやすくする機能が[bmaltais氏のリポジトリ](https://github.com/bmaltais/kohya_ss)で提供されています（英語です）のであわせてご覧ください。bmaltais氏に感謝します。
+
+以下のスクリプトがあります。
+
+* DreamBooth、U-NetおよびText Encoderの学習をサポート
+* fine-tuning、同上
+* 画像生成
+* モデル変換（Stable Diffision ckpt/safetensorsとDiffusersの相互変換）
+
+## 使用法について
+
+当リポジトリ内およびnote.comに記事がありますのでそちらをご覧ください（将来的にはすべてこちらへ移すかもしれません）。
+
+* [学習について、共通編](./docs/train_README-ja.md) : データ整備やオプションなど
+    * [データセット設定](./docs/config_README-ja.md)
+* [DreamBoothの学習について](./docs/train_db_README-ja.md)
+* [fine-tuningのガイド](./docs/fine_tune_README_ja.md):
+* [LoRAの学習について](./docs/train_network_README-ja.md)
+* [Textual Inversionの学習について](./docs/train_ti_README-ja.md)
+* [画像生成スクリプト](./docs/gen_img_README-ja.md)
+* note.com [モデル変換スクリプト](https://note.com/kohya_ss/n/n374f316fe4ad)
+
+## Windowsでの動作に必要なプログラム
+
+Python 3.10.6およびGitが必要です。
+
+- Python 3.10.6: https://www.python.org/ftp/python/3.10.6/python-3.10.6-amd64.exe
+- git: https://git-scm.com/download/win
+
+PowerShellを使う場合、venvを使えるようにするためには以下の手順でセキュリティ設定を変更してください。
+（venvに限らずスクリプトの実行が可能になりますので注意してください。）
+
+- PowerShellを管理者として開きます。
+- 「Set-ExecutionPolicy Unrestricted」と入力し、Yと答えます。
+- 管理者のPowerShellを閉じます。
+
+## Windows環境でのインストール
+
+以下の例ではPyTorchは1.12.1／CUDA 11.6版をインストールします。CUDA 11.3版やPyTorch 1.13を使う場合は適宜書き換えください。
+
+（なお、python -m venv～の行で「python」とだけ表示された場合、py -m venv～のようにpythonをpyに変更してください。）
+
+通常の（管理者ではない）PowerShellを開き以下を順に実行します。
+
+```powershell
+git clone https://github.com/kohya-ss/sd-scripts.git
+cd sd-scripts
+
+python -m venv venv
+.\venv\Scripts\activate
+
+pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116
+pip install --upgrade -r requirements.txt
+pip install -U -I --no-deps https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl
+
+cp .\bitsandbytes_windows\*.dll .\venv\Lib\site-packages\bitsandbytes\
+cp .\bitsandbytes_windows\cextension.py .\venv\Lib\site-packages\bitsandbytes\cextension.py
+cp .\bitsandbytes_windows\main.py .\venv\Lib\site-packages\bitsandbytes\cuda_setup\main.py
+
+accelerate config
+```
+
+<!-- 
+pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117
+pip install --use-pep517 --upgrade -r requirements.txt
+pip install -U -I --no-deps xformers==0.0.16
+-->
+
+コマンドプロンプトでは以下になります。
+
+
+```bat
+git clone https://github.com/kohya-ss/sd-scripts.git
+cd sd-scripts
+
+python -m venv venv
+.\venv\Scripts\activate
+
+pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116
+pip install --upgrade -r requirements.txt
+pip install -U -I --no-deps https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl
+
+copy /y .\bitsandbytes_windows\*.dll .\venv\Lib\site-packages\bitsandbytes\
+copy /y .\bitsandbytes_windows\cextension.py .\venv\Lib\site-packages\bitsandbytes\cextension.py
+copy /y .\bitsandbytes_windows\main.py .\venv\Lib\site-packages\bitsandbytes\cuda_setup\main.py
+
+accelerate config
+```
+
+（注:``python -m venv venv`` のほうが ``python -m venv --system-site-packages venv`` より安全そうなため書き換えました。globalなpythonにパッケージがインストールしてあると、後者だといろいろと問題が起きます。）
+
+accelerate configの質問には以下のように答えてください。（bf16で学習する場合、最後の質問にはbf16と答えてください。）
+
+※0.15.0から日本語環境では選択のためにカーソルキーを押すと落ちます（……）。数字キーの0、1、2……で選択できますので、そちらを使ってください。
+
+```txt
+- This machine
+- No distributed training
+- NO
+- NO
+- NO
+- all
+- fp16
+```
+
+※場合によって ``ValueError: fp16 mixed precision requires a GPU`` というエラーが出ることがあるようです。この場合、6番目の質問（
+``What GPU(s) (by id) should be used for training on this machine as a comma-separated list? [all]:``）に「0」と答えてください。（id `0`のGPUが使われます。）
+
+### PyTorchとxformersのバージョンについて
+
+他のバージョンでは学習がうまくいかない場合があるようです。特に他の理由がなければ指定のバージョンをお使いください。
+
+### オプション：Lion8bitを使う
+
+Lion8bitを使う場合には`bitsandbytes`を0.38.0以降にアップグレードする必要があります。`bitsandbytes`をアンインストールし、Windows環境では例えば[こちら](https://github.com/jllllll/bitsandbytes-windows-webui)などからWindows版のwhlファイルをインストールしてください。たとえば以下のような手順になります。
+
+```powershell
+pip install https://github.com/jllllll/bitsandbytes-windows-webui/raw/main/bitsandbytes-0.38.1-py3-none-any.whl
+```
+
+アップグレード時には`pip install .`でこのリポジトリを更新し、必要に応じて他のパッケージもアップグレードしてください。
+
+## アップグレード
+
+新しいリリースがあった場合、以下のコマンドで更新できます。
+
+```powershell
+cd sd-scripts
+git pull
+.\venv\Scripts\activate
+pip install --use-pep517 --upgrade -r requirements.txt
+```
+
+コマンドが成功すれば新しいバージョンが使用できます。
+
+## 謝意
+
+LoRAの実装は[cloneofsimo氏のリポジトリ](https://github.com/cloneofsimo/lora)を基にしたものです。感謝申し上げます。
+
+Conv2d 3x3への拡大は [cloneofsimo氏](https://github.com/cloneofsimo/lora) が最初にリリースし、KohakuBlueleaf氏が [LoCon](https://github.com/KohakuBlueleaf/LoCon) でその有効性を明らかにしたものです。KohakuBlueleaf氏に深く感謝します。
+
+## ライセンス
+
+スクリプトのライセンスはASL 2.0ですが（Diffusersおよびcloneofsimo氏のリポジトリ由来のものも同様）、一部他のライセンスのコードを含みます。
+
+[Memory Efficient Attention Pytorch](https://github.com/lucidrains/memory-efficient-attention-pytorch): MIT
+
+[bitsandbytes](https://github.com/TimDettmers/bitsandbytes): MIT
+
+[BLIP](https://github.com/salesforce/BLIP): BSD-3-Clause
+
+
diff --git a/README.md b/README.md
index 58ed257a9ff549ebf60e968a0ee12f8c672faf3f..67ebc0fb67211b9fafeff7e144b7506b721ca829 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,475 @@
 ---
-title: Kohya Ss
-emoji: 📉
-colorFrom: purple
-colorTo: green
+title: kohya_ss
+app_file: kohya_gui.py
 sdk: gradio
-sdk_version: 3.40.1
-app_file: app.py
-pinned: false
+sdk_version: 3.36.1
 ---
+# Kohya's GUI
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+This repository mostly provides a Windows-focused Gradio GUI for [Kohya's Stable Diffusion trainers](https://github.com/kohya-ss/sd-scripts)... but support for Linux OS is also provided through community contributions. Macos is not great at the moment.
+
+The GUI allows you to set the training parameters and generate and run the required CLI commands to train the model.
+
+## Table of Contents
+
+1. [Tutorials](#tutorials)
+2. [Installation](#installation)
+   1. [Windows](#windows)
+      1. [Windows Pre-requirements](#windows-pre-requirements)
+      2. [Setup](#setup)
+      3. [Optional: CUDNN 8.6](#optional-cudnn-86)
+   2. [Linux and macOS](#linux-and-macos)
+      1. [Linux Pre-requirements](#linux-pre-requirements)
+      2. [Setup](#setup-1)
+      3. [Install Location](#install-location)
+   3. [Runpod](#runpod)
+   4. [Docker](#docker)
+3. [Upgrading](#upgrading)
+   1. [Windows Upgrade](#windows-upgrade)
+   2. [Linux and macOS Upgrade](#linux-and-macos-upgrade)
+4. [Starting GUI Service](#starting-gui-service)
+   1. [Launching the GUI on Windows](#launching-the-gui-on-windows)
+   2. [Launching the GUI on Linux and macOS](#launching-the-gui-on-linux-and-macos)
+5. [Dreambooth](#dreambooth)
+6. [Finetune](#finetune)
+7. [Train Network](#train-network)
+8. [LoRA](#lora)
+9. [Sample image generation during training](#sample-image-generation-during-training)
+10. [Troubleshooting](#troubleshooting)
+   1. [Page File Limit](#page-file-limit)
+   2. [No module called tkinter](#no-module-called-tkinter)
+   3. [FileNotFoundError](#filenotfounderror)
+11. [Change History](#change-history)
+
+## Tutorials
+
+[How to Create a LoRA Part 1: Dataset Preparation](https://www.youtube.com/watch?v=N4_-fB62Hwk):
+
+[![LoRA Part 1 Tutorial](https://img.youtube.com/vi/N4_-fB62Hwk/0.jpg)](https://www.youtube.com/watch?v=N4_-fB62Hwk)
+
+[How to Create a LoRA Part 2: Training the Model](https://www.youtube.com/watch?v=k5imq01uvUY):
+### About SDXL training
+
+The feature of SDXL training is now available in sdxl branch as an experimental feature. 
+
+Summary of the feature:
+
+- `sdxl_train.py` is a script for SDXL fine-tuning. The usage is almost the same as `fine_tune.py`, but it also supports DreamBooth dataset.
+  - `--full_bf16` option is added. Thanks to KohakuBlueleaf!
+    - This option enables the full bfloat16 training (includes gradients). This option is useful to reduce the GPU memory usage. 
+    - However, bitsandbytes==0.35 doesn't seem to support this. Please use a newer version of bitsandbytes or another optimizer.
+    - I cannot find bitsandbytes>0.35.0 that works correctly on Windows.
+    - In addition, the full bfloat16 training might be unstable. Please use it at your own risk.
+- `prepare_buckets_latents.py` now supports SDXL fine-tuning.
+- `sdxl_train_network.py` is a script for LoRA training for SDXL. The usage is almost the same as `train_network.py`.
+- Both scripts has following additional options:
+  - `--cache_text_encoder_outputs`: Cache the outputs of the text encoders. This option is useful to reduce the GPU memory usage. This option cannot be used with options for shuffling or dropping the captions.
+  - `--no_half_vae`: Disable the half-precision (mixed-precision) VAE. VAE for SDXL seems to produce NaNs in some cases. This option is useful to avoid the NaNs.
+- The image generation during training is now available. However, the VAE for SDXL seems to produce NaNs in some cases when using `fp16`. The images will be black. Currently, the NaNs cannot be avoided even with `--no_half_vae` option. It works with `bf16` or without mixed precision.
+
+- `--weighted_captions` option is not supported yet for both scripts.
+- `--min_timestep` and `--max_timestep` options are added to each training script. These options can be used to train U-Net with different timesteps. The default values are 0 and 1000.
+
+- `sdxl_train_textual_inversion.py` is a script for Textual Inversion training for SDXL. The usage is almost the same as `train_textual_inversion.py`.
+  - `--cache_text_encoder_outputs` is not supported.
+  - `token_string` must be alphabet only currently, due to the limitation of the open-clip tokenizer.
+  - There are two options for captions:
+    1. Training with captions. All captions must include the token string. The token string is replaced with multiple tokens.
+    2. Use `--use_object_template` or `--use_style_template` option. The captions are generated from the template. The existing captions are ignored.
+  - See below for the format of the embeddings.
+  
+- `sdxl_gen_img.py` is added. This script can be used to generate images with SDXL, including LoRA. See the help message for the usage.
+  - Textual Inversion is supported, but the name for the embeds in the caption becomes alphabet only. For example, `neg_hand_v1.safetensors` can be activated with `neghandv`.
+
+`requirements.txt` is updated to support SDXL training. 
+
+#### Tips for SDXL training
+
+- The default resolution of SDXL is 1024x1024.
+- The fine-tuning can be done with 24GB GPU memory with the batch size of 1. For 24GB GPU, the following options are recommended:
+  - Train U-Net only.
+  - Use gradient checkpointing.
+  - Use `--cache_text_encoder_outputs` option and caching latents.
+  - Use Adafactor optimizer. RMSprop 8bit or Adagrad 8bit may work. AdamW 8bit doesn't seem to work.
+- The LoRA training can be done with 12GB GPU memory.
+- `--network_train_unet_only` option is highly recommended for SDXL LoRA. Because SDXL has two text encoders, the result of the training will be unexpected.
+- PyTorch 2 seems to use slightly less GPU memory than PyTorch 1.
+- `--bucket_reso_steps` can be set to 32 instead of the default value 64. Smaller values than 32 will not work for SDXL training.
+
+Example of the optimizer settings for Adafactor with the fixed learning rate:
+```toml
+optimizer_type = "adafactor"
+optimizer_args = [ "scale_parameter=False", "relative_step=False", "warmup_init=False" ]
+lr_scheduler = "constant_with_warmup"
+lr_warmup_steps = 100
+learning_rate = 4e-7 # SDXL original learning rate
+```
+
+### Format of Textual Inversion embeddings
+
+```python
+from safetensors.torch import save_file
+
+state_dict = {"clip_g": embs_for_text_encoder_1280, "clip_l": embs_for_text_encoder_768}
+save_file(state_dict, file)
+```
+
+### TODO
+
+- [ ] Support conversion of Diffusers SDXL models.
+- [ ] Support `--weighted_captions` option.
+- [ ] Change `--output_config` option to continue the training.
+- [ ] Extend `--full_bf16` for all the scripts.
+- [x] Support Textual Inversion training.
+
+## About requirements.txt
+
+[![LoRA Part 2 Tutorial](https://img.youtube.com/vi/k5imq01uvUY/0.jpg)](https://www.youtube.com/watch?v=k5imq01uvUY)
+
+Newer Tutorial: [Generate Studio Quality Realistic Photos By Kohya LoRA Stable Diffusion Training](https://www.youtube.com/watch?v=TpuDOsuKIBo):
+The scripts are tested with PyTorch 1.12.1 and 2.0.1, Diffusers 0.17.1.
+
+[![Newer Tutorial: Generate Studio Quality Realistic Photos By Kohya LoRA Stable Diffusion Training](https://user-images.githubusercontent.com/19240467/235306147-85dd8126-f397-406b-83f2-368927fa0281.png)](https://www.youtube.com/watch?v=TpuDOsuKIBo)
+
+Newer Tutorial: [How To Install And Use Kohya LoRA GUI / Web UI on RunPod IO](https://www.youtube.com/watch?v=3uzCNrQao3o):
+
+[![How To Install And Use Kohya LoRA GUI / Web UI on RunPod IO With Stable Diffusion & Automatic1111](https://github-production-user-asset-6210df.s3.amazonaws.com/19240467/238678226-0c9c3f7d-c308-4793-b790-999fdc271372.png)](https://www.youtube.com/watch?v=3uzCNrQao3o)
+
+## Installation
+
+### Windows
+
+#### Windows Pre-requirements
+
+To install the necessary dependencies on a Windows system, follow these steps:
+
+1. Install [Python 3.10](https://www.python.org/ftp/python/3.10.9/python-3.10.9-amd64.exe).
+   - During the installation process, ensure that you select the option to add Python to the 'PATH' environment variable.
+
+2. Install [Git](https://git-scm.com/download/win).
+
+3. Install the [Visual Studio 2015, 2017, 2019, and 2022 redistributable](https://aka.ms/vs/17/release/vc_redist.x64.exe).
+
+#### Setup
+
+To set up the project, follow these steps:
+
+1. Open a terminal and navigate to the desired installation directory.
+
+2. Clone the repository by running the following command:
+   ```
+   git clone https://github.com/bmaltais/kohya_ss.git
+   ```
+
+3. Change into the `kohya_ss` directory:
+   ```
+   cd kohya_ss
+   ```
+
+4. Run the setup script by executing the following command:
+   ```
+   .\setup.bat
+   ```
+
+#### Optional: CUDNN 8.6
+
+The following steps are optional but can improve the learning speed for owners of NVIDIA 30X0/40X0 GPUs. These steps enable larger training batch sizes and faster training speeds.
+
+Please note that the CUDNN 8.6 DLLs needed for this process cannot be hosted on GitHub due to file size limitations. You can download them [here](https://github.com/bmaltais/python-library/raw/main/cudnn_windows.zip) to boost sample generation speed (almost 50% on a 4090 GPU). After downloading the ZIP file, follow the installation steps below:
+
+1. Unzip the downloaded file and place the `cudnn_windows` folder in the root directory of the `kohya_ss` repository.
+
+2. Run .\setup.bat and select the option to install cudann.
+
+### Linux and macOS
+
+#### Linux Pre-requirements
+
+To install the necessary dependencies on a Linux system, ensure that you fulfill the following requirements:
+
+- Ensure that `venv` support is pre-installed. You can install it on Ubuntu 22.04 using the command:
+  ```
+  apt install python3.10-venv
+  ```
+
+- Install the cudaNN drivers by following the instructions provided in [this link](https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64).
+
+- Make sure you have Python version 3.10.6 or higher (but lower than 3.11.0) installed on your system.
+
+- If you are using WSL2, set the `LD_LIBRARY_PATH` environment variable by executing the following command:
+  ```
+  export LD_LIBRARY_PATH=/usr/lib/wsl/lib/
+  ```
+
+#### Setup
+
+To set up the project on Linux or macOS, perform the following steps:
+
+1. Open a terminal and navigate to the desired installation directory.
+
+2. Clone the repository by running the following command:
+   ```
+   git clone https://github.com/bmaltais/kohya_ss.git
+   ```
+
+3. Change into the `kohya_ss` directory:
+   ```
+   cd kohya_ss
+   ```
+
+4. If you encounter permission issues, make the `setup.sh` script executable by running the following command:
+   ```
+   chmod +x ./setup.sh
+   ```
+
+5. Run the setup script by executing the following command:
+   ```
+   ./setup.sh
+   ```
+
+   Note: If you need additional options or information about the runpod environment, you can use `setup.sh -h` or `setup.sh --help` to display the help message.
+
+#### Install Location
+
+The default installation location on Linux is the directory where the script is located. If a previous installation is detected in that location, the setup will proceed there. Otherwise, the installation will fall back to `/opt/kohya_ss`. If `/opt` is not writable, the fallback location will be `$HOME/kohya_ss`. Finally, if none of the previous options are viable, the installation will be performed in the current directory.
+
+For macOS and other non-Linux systems, the installation process will attempt to detect the previous installation directory based on where the script is run. If a previous installation is not found, the default location will be `$HOME/kohya_ss`. You can override this behavior by specifying a custom installation directory using the `-d` or `--dir` option when running the setup script.
+
+If you choose to use the interactive mode, the default values for the accelerate configuration screen will be "This machine," "None," and "No" for the remaining questions. These default answers are the same as the Windows installation.
+
+### Runpod
+#### Manual installation
+
+To install the necessary components for Runpod and run kohya_ss, follow these steps:
+
+1. Select the Runpod pytorch 2.0.1 template. This is important. Other templates may not work.
+
+2. SSH into the Runpod.
+
+3. Clone the repository by running the following command:
+   ```
+   cd /workspace
+   git clone https://github.com/bmaltais/kohya_ss.git
+   ```
+
+4. Run the setup script:
+   ```
+   cd kohya_ss
+   ./setup-runpod.sh
+   ```
+
+5. Run the gui with:
+   ```
+   ./gui.sh --share --headless
+   ```
+
+   or with this if you expose 7860 directly via the runpod configuration
+
+   ```
+   ./gui.sh --listen=0.0.0.0 --headless
+   ```
+
+6. Connect to the public URL displayed after the installation process is completed.
+
+#### Pre-built Runpod template
+
+To run from a pre-built Runpod template you can:
+
+1. Open the Runpod template by clicking on https://runpod.io/gsc?template=ya6013lj5a&ref=w18gds2n
+
+2. Deploy the template on the desired host
+
+3. Once deployed connect to the Runpod on HTTP 3010 to connect to kohya_ss GUI. You can also connect to auto1111 on HTTP 3000.
+
+
+### Docker
+#### Local docker build
+
+If you prefer to use Docker, follow the instructions below:
+
+1. Ensure that you have Git and Docker installed on your Windows or Linux system.
+
+2. Open your OS shell (Command Prompt or Terminal) and run the following commands:
+
+   ```bash
+   git clone https://github.com/bmaltais/kohya_ss.git
+   cd kohya_ss
+   docker compose build
+   docker compose run --service-ports kohya-ss-gui
+   ```
+
+   Note: The initial run may take up to 20 minutes to complete.
+
+   Please be aware of the following limitations when using Docker:
+
+   - All training data must be placed in the `dataset` subdirectory, as the Docker container cannot access files from other directories.
+   - The file picker feature is not functional. You need to manually set the folder path and config file path.
+   - Dialogs may not work as expected, and it is recommended to use unique file names to avoid conflicts.
+   - There is no built-in auto-update support. To update the system, you must run update scripts outside of Docker and rebuild using `docker compose build`.
+
+   If you are running Linux, an alternative Docker container port with fewer limitations is available [here](https://github.com/P2Enjoy/kohya_ss-docker).
+
+#### ashleykleynhans runpod docker builds
+
+You may want to use the following Dockerfile repos to build the images:
+
+   - Standalone Kohya_ss template: https://github.com/ashleykleynhans/kohya-docker
+   - Auto1111 + Kohya_ss GUI template: https://github.com/ashleykleynhans/stable-diffusion-docker
+
+## Upgrading
+
+To upgrade your installation to a new version, follow the instructions below.
+
+### Windows Upgrade
+
+If a new release becomes available, you can upgrade your repository by running the following commands from the root directory of the project:
+
+1. Pull the latest changes from the repository:
+   ```powershell
+   git pull
+   ```
+
+2. Run the setup script:
+   ```powershell
+   .\setup.bat
+   ```
+
+### Linux and macOS Upgrade
+
+To upgrade your installation on Linux or macOS, follow these steps:
+
+1. Open a terminal and navigate to the root
+
+ directory of the project.
+
+2. Pull the latest changes from the repository:
+   ```bash
+   git pull
+   ```
+
+3. Refresh and update everything:
+   ```bash
+   ./setup.sh
+   ```
+
+## Starting GUI Service
+
+To launch the GUI service, you can use the provided scripts or run the `kohya_gui.py` script directly. Use the command line arguments listed below to configure the underlying service.
+
+```text
+--listen: Specify the IP address to listen on for connections to Gradio.
+--username: Set a username for authentication.
+--password: Set a password for authentication.
+--server_port: Define the port to run the server listener on.
+--inbrowser: Open the Gradio UI in a web browser.
+--share: Share the Gradio UI.
+```
+
+### Launching the GUI on Windows
+
+On Windows, you can use either the `gui.ps1` or `gui.bat` script located in the root directory. Choose the script that suits your preference and run it in a terminal, providing the desired command line arguments. Here's an example:
+
+```powershell
+gui.ps1 --listen 127.0.0.1 --server_port 7860 --inbrowser --share
+```
+
+or
+
+```powershell
+gui.bat --listen 127.0.0.1 --server_port 7860 --inbrowser --share
+```
+
+### Launching the GUI on Linux and macOS
+
+To launch the GUI on Linux or macOS, run the `gui.sh` script located in the root directory. Provide the desired command line arguments as follows:
+
+```bash
+gui.sh --listen 127.0.0.1 --server_port 7860 --inbrowser --share
+```
+
+## Dreambooth
+
+For specific instructions on using the Dreambooth solution, please refer to the [Dreambooth README](https://github.com/bmaltais/kohya_ss/blob/master/train_db_README.md).
+
+## Finetune
+
+For specific instructions on using the Finetune solution, please refer to the [Finetune README](https://github.com/bmaltais/kohya_ss/blob/master/fine_tune_README.md).
+
+## Train Network
+
+For specific instructions on training a network, please refer to the [Train network README](https://github.com/bmaltais/kohya_ss/blob/master/train_network_README.md).
+
+## LoRA
+
+To train a LoRA, you can currently use the `train_network.py` code. You can create a LoRA network by using the all-in-one GUI.
+
+Once you have created the LoRA network, you can generate images using auto1111 by installing [this extension](https://github.com/kohya-ss/sd-webui-additional-networks).
+
+The following are the names of LoRA types used in this repository:
+
+1. LoRA-LierLa: LoRA for Linear layers and Conv2d layers with a 1x1 kernel.
+
+2. LoRA-C3Lier: LoRA for Conv2d layers with a 3x3 kernel, in addition to LoRA-LierLa.
+
+LoRA-LierLa is the default LoRA type for `train_network.py` (without `conv_dim` network argument). You can use LoRA-LierLa with our extension for AUTOMATIC1111's Web UI or the built-in LoRA feature of the Web UI.
+
+To use LoRA-C3Lier with the Web UI, please use our extension.
+
+## Sample image generation during training
+
+A prompt file might look like this, for example:
+
+```
+# prompt 1
+masterpiece, best quality, (1girl), in white shirts, upper body, looking at viewer, simple background --n low quality, worst quality, bad anatomy, bad composition, poor, low effort --w 768 --h 768 --d 1 --l 7.5 --s 28
+
+# prompt 2
+masterpiece, best quality, 1boy, in business suit, standing at street, looking back --n (low quality, worst quality), bad anatomy, bad composition, poor, low effort --w 576 --h 832 --d 2 --l 5.5 --s 40
+```
+
+Lines beginning with `#` are comments. You can specify options for the generated image with options like `--n` after the prompt. The following options can be used:
+
+- `--n`: Negative prompt up to the next option.
+- `--w`: Specifies the width of the generated image.
+- `--h`: Specifies the height of the generated image.
+- `--d`: Specifies the seed of the generated image.
+- `--l`: Specifies the CFG scale of the generated image.
+- `--s`: Specifies the number of steps in the generation.
+
+The prompt weighting such as `( )` and `[ ]` are working.
+
+## Troubleshooting
+
+If you encounter any issues, refer to the troubleshooting steps below.
+
+### Page File Limit
+
+If you encounter an X error related to the page file, you may need to increase the page file size limit in Windows.
+
+### No module called tkinter
+
+If you encounter an error indicating that the module `tkinter` is not found, try reinstalling Python 3.10 on your system.
+
+### FileNotFoundError
+
+If you come across a `FileNotFoundError`, it is likely due to an installation issue. Make sure you do not have any locally installed Python modules that could conflict with the ones installed in the virtual environment. You can uninstall them by following these steps:
+
+1. Open a new PowerShell terminal and ensure that no virtual environment is active.
+
+2. Run the following commands to create a backup file of your locally installed pip packages and then uninstall them:
+   ```powershell
+   pip freeze > uninstall.txt
+   pip uninstall -r uninstall.txt
+   ```
+
+   After uninstalling the local packages, redo the installation steps within the `kohya_ss` virtual environment.
+
+## Change History
+
+* 2023/07/11 (v21.8.2)
+  - Let Tensorboard works in docker #1137
+  - Fix for accelerate issue
+  - Add SDXL TI training support
+  - Rework gui for common layout
+  - More LoRA tools to class
+  - Add no_half_vae option to TI
\ No newline at end of file
diff --git a/XTI_hijack.py b/XTI_hijack.py
new file mode 100644
index 0000000000000000000000000000000000000000..36b5d3f2b501aca8dbb83b40d1697c87d435c170
--- /dev/null
+++ b/XTI_hijack.py
@@ -0,0 +1,201 @@
+import torch
+from typing import Union, List, Optional, Dict, Any, Tuple
+from diffusers.models.unet_2d_condition import UNet2DConditionOutput
+
+from library.original_unet import SampleOutput
+
+
+def unet_forward_XTI(
+    self,
+    sample: torch.FloatTensor,
+    timestep: Union[torch.Tensor, float, int],
+    encoder_hidden_states: torch.Tensor,
+    class_labels: Optional[torch.Tensor] = None,
+    return_dict: bool = True,
+) -> Union[Dict, Tuple]:
+    r"""
+    Args:
+        sample (`torch.FloatTensor`): (batch, channel, height, width) noisy inputs tensor
+        timestep (`torch.FloatTensor` or `float` or `int`): (batch) timesteps
+        encoder_hidden_states (`torch.FloatTensor`): (batch, sequence_length, feature_dim) encoder hidden states
+        return_dict (`bool`, *optional*, defaults to `True`):
+            Whether or not to return a dict instead of a plain tuple.
+
+    Returns:
+        `SampleOutput` or `tuple`:
+        `SampleOutput` if `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is the sample tensor.
+    """
+    # By default samples have to be AT least a multiple of the overall upsampling factor.
+    # The overall upsampling factor is equal to 2 ** (# num of upsampling layears).
+    # However, the upsampling interpolation output size can be forced to fit any upsampling size
+    # on the fly if necessary.
+    # デフォルトではサンプルは「2^アップサンプルの数」、つまり64の倍数である必要がある
+    # ただそれ以外のサイズにも対応できるように、必要ならアップサンプルのサイズを変更する
+    # 多分画質が悪くなるので、64で割り切れるようにしておくのが良い
+    default_overall_up_factor = 2**self.num_upsamplers
+
+    # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
+    # 64で割り切れないときはupsamplerにサイズを伝える
+    forward_upsample_size = False
+    upsample_size = None
+
+    if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):
+        # logger.info("Forward upsample size to force interpolation output size.")
+        forward_upsample_size = True
+
+    # 1. time
+    timesteps = timestep
+    timesteps = self.handle_unusual_timesteps(sample, timesteps)  # 変な時だけ処理
+
+    t_emb = self.time_proj(timesteps)
+
+    # timesteps does not contain any weights and will always return f32 tensors
+    # but time_embedding might actually be running in fp16. so we need to cast here.
+    # there might be better ways to encapsulate this.
+    # timestepsは重みを含まないので常にfloat32のテンソルを返す
+    # しかしtime_embeddingはfp16で動いているかもしれないので、ここでキャストする必要がある
+    # time_projでキャストしておけばいいんじゃね？
+    t_emb = t_emb.to(dtype=self.dtype)
+    emb = self.time_embedding(t_emb)
+
+    # 2. pre-process
+    sample = self.conv_in(sample)
+
+    # 3. down
+    down_block_res_samples = (sample,)
+    down_i = 0
+    for downsample_block in self.down_blocks:
+        # downblockはforwardで必ずencoder_hidden_statesを受け取るようにしても良さそうだけど、
+        # まあこちらのほうがわかりやすいかもしれない
+        if downsample_block.has_cross_attention:
+            sample, res_samples = downsample_block(
+                hidden_states=sample,
+                temb=emb,
+                encoder_hidden_states=encoder_hidden_states[down_i : down_i + 2],
+            )
+            down_i += 2
+        else:
+            sample, res_samples = downsample_block(hidden_states=sample, temb=emb)
+
+        down_block_res_samples += res_samples
+
+    # 4. mid
+    sample = self.mid_block(sample, emb, encoder_hidden_states=encoder_hidden_states[6])
+
+    # 5. up
+    up_i = 7
+    for i, upsample_block in enumerate(self.up_blocks):
+        is_final_block = i == len(self.up_blocks) - 1
+
+        res_samples = down_block_res_samples[-len(upsample_block.resnets) :]
+        down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]  # skip connection
+
+        # if we have not reached the final block and need to forward the upsample size, we do it here
+        # 前述のように最後のブロック以外ではupsample_sizeを伝える
+        if not is_final_block and forward_upsample_size:
+            upsample_size = down_block_res_samples[-1].shape[2:]
+
+        if upsample_block.has_cross_attention:
+            sample = upsample_block(
+                hidden_states=sample,
+                temb=emb,
+                res_hidden_states_tuple=res_samples,
+                encoder_hidden_states=encoder_hidden_states[up_i : up_i + 3],
+                upsample_size=upsample_size,
+            )
+            up_i += 3
+        else:
+            sample = upsample_block(
+                hidden_states=sample, temb=emb, res_hidden_states_tuple=res_samples, upsample_size=upsample_size
+            )
+
+    # 6. post-process
+    sample = self.conv_norm_out(sample)
+    sample = self.conv_act(sample)
+    sample = self.conv_out(sample)
+
+    if not return_dict:
+        return (sample,)
+
+    return SampleOutput(sample=sample)
+
+
+def downblock_forward_XTI(
+    self, hidden_states, temb=None, encoder_hidden_states=None, attention_mask=None, cross_attention_kwargs=None
+):
+    output_states = ()
+    i = 0
+
+    for resnet, attn in zip(self.resnets, self.attentions):
+        if self.training and self.gradient_checkpointing:
+
+            def create_custom_forward(module, return_dict=None):
+                def custom_forward(*inputs):
+                    if return_dict is not None:
+                        return module(*inputs, return_dict=return_dict)
+                    else:
+                        return module(*inputs)
+
+                return custom_forward
+
+            hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
+            hidden_states = torch.utils.checkpoint.checkpoint(
+                create_custom_forward(attn, return_dict=False), hidden_states, encoder_hidden_states[i]
+            )[0]
+        else:
+            hidden_states = resnet(hidden_states, temb)
+            hidden_states = attn(hidden_states, encoder_hidden_states=encoder_hidden_states[i]).sample
+
+        output_states += (hidden_states,)
+        i += 1
+
+    if self.downsamplers is not None:
+        for downsampler in self.downsamplers:
+            hidden_states = downsampler(hidden_states)
+
+        output_states += (hidden_states,)
+
+    return hidden_states, output_states
+
+
+def upblock_forward_XTI(
+    self,
+    hidden_states,
+    res_hidden_states_tuple,
+    temb=None,
+    encoder_hidden_states=None,
+    upsample_size=None,
+):
+    i = 0
+    for resnet, attn in zip(self.resnets, self.attentions):
+        # pop res hidden states
+        res_hidden_states = res_hidden_states_tuple[-1]
+        res_hidden_states_tuple = res_hidden_states_tuple[:-1]
+        hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
+
+        if self.training and self.gradient_checkpointing:
+
+            def create_custom_forward(module, return_dict=None):
+                def custom_forward(*inputs):
+                    if return_dict is not None:
+                        return module(*inputs, return_dict=return_dict)
+                    else:
+                        return module(*inputs)
+
+                return custom_forward
+
+            hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
+            hidden_states = torch.utils.checkpoint.checkpoint(
+                create_custom_forward(attn, return_dict=False), hidden_states, encoder_hidden_states[i]
+            )[0]
+        else:
+            hidden_states = resnet(hidden_states, temb)
+            hidden_states = attn(hidden_states, encoder_hidden_states=encoder_hidden_states[i]).sample
+
+        i += 1
+
+    if self.upsamplers is not None:
+        for upsampler in self.upsamplers:
+            hidden_states = upsampler(hidden_states, upsample_size)
+
+    return hidden_states
diff --git a/__pycache__/dreambooth_gui.cpython-310.pyc b/__pycache__/dreambooth_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3be2775a9cfe4796d3f335b56d2cc2173cd5e3bb
Binary files /dev/null and b/__pycache__/dreambooth_gui.cpython-310.pyc differ
diff --git a/__pycache__/finetune_gui.cpython-310.pyc b/__pycache__/finetune_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..865d2283158a61c90adfa06562bd913a2d873a58
Binary files /dev/null and b/__pycache__/finetune_gui.cpython-310.pyc differ
diff --git a/__pycache__/lora_gui.cpython-310.pyc b/__pycache__/lora_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f5307cb60fa7d48bdf22b476699208c8b0a3cbbc
Binary files /dev/null and b/__pycache__/lora_gui.cpython-310.pyc differ
diff --git a/__pycache__/textual_inversion_gui.cpython-310.pyc b/__pycache__/textual_inversion_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cdcc9a4e9745def47584b5296c648c77f580accd
Binary files /dev/null and b/__pycache__/textual_inversion_gui.cpython-310.pyc differ
diff --git a/_typos.toml b/_typos.toml
new file mode 100644
index 0000000000000000000000000000000000000000..4902a59b4270e00984877f3637ce9a06a0dd85b5
--- /dev/null
+++ b/_typos.toml
@@ -0,0 +1,15 @@
+# Files for typos
+# Instruction:  https://github.com/marketplace/actions/typos-action#getting-started
+
+[default.extend-identifiers]
+
+[default.extend-words]
+NIN="NIN"
+parms="parms"
+nin="nin"
+extention="extention" # Intentionally left
+nd="nd"
+
+
+[files]
+extend-exclude = ["_typos.toml"]
\ No newline at end of file
diff --git a/activate.ps1 b/activate.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..ae3888be3b2f4a041d6ad20178a41e61be856701
--- /dev/null
+++ b/activate.ps1
@@ -0,0 +1 @@
+.\venv\Scripts\activate
\ No newline at end of file
diff --git a/bitsandbytes_windows/cextension.py b/bitsandbytes_windows/cextension.py
new file mode 100644
index 0000000000000000000000000000000000000000..d38684a2038bc598d2a8f6f0791217598891de82
--- /dev/null
+++ b/bitsandbytes_windows/cextension.py
@@ -0,0 +1,54 @@
+import ctypes as ct
+from pathlib import Path
+from warnings import warn
+
+from .cuda_setup.main import evaluate_cuda_setup
+
+
+class CUDALibrary_Singleton(object):
+    _instance = None
+
+    def __init__(self):
+        raise RuntimeError("Call get_instance() instead")
+
+    def initialize(self):
+        binary_name = evaluate_cuda_setup()
+        package_dir = Path(__file__).parent
+        binary_path = package_dir / binary_name
+
+        if not binary_path.exists():
+            print(f"CUDA SETUP: TODO: compile library for specific version: {binary_name}")
+            legacy_binary_name = "libbitsandbytes.so"
+            print(f"CUDA SETUP: Defaulting to {legacy_binary_name}...")
+            binary_path = package_dir / legacy_binary_name
+            if not binary_path.exists():
+                print('CUDA SETUP: CUDA detection failed. Either CUDA driver not installed, CUDA not installed, or you have multiple conflicting CUDA libraries!')
+                print('CUDA SETUP: If you compiled from source, try again with `make CUDA_VERSION=DETECTED_CUDA_VERSION` for example, `make CUDA_VERSION=113`.')
+                raise Exception('CUDA SETUP: Setup Failed!')
+            # self.lib = ct.cdll.LoadLibrary(binary_path)
+            self.lib = ct.cdll.LoadLibrary(str(binary_path))            # $$$
+        else:
+            print(f"CUDA SETUP: Loading binary {binary_path}...")
+            # self.lib = ct.cdll.LoadLibrary(binary_path)
+            self.lib = ct.cdll.LoadLibrary(str(binary_path))            # $$$
+
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = cls.__new__(cls)
+            cls._instance.initialize()
+        return cls._instance
+
+
+lib = CUDALibrary_Singleton.get_instance().lib
+try:
+    lib.cadam32bit_g32
+    lib.get_context.restype = ct.c_void_p
+    lib.get_cusparse.restype = ct.c_void_p
+    COMPILED_WITH_CUDA = True
+except AttributeError:
+    warn(
+        "The installed version of bitsandbytes was compiled without GPU support. "
+        "8-bit optimizers and GPU quantization are unavailable."
+    )
+    COMPILED_WITH_CUDA = False
diff --git a/bitsandbytes_windows/libbitsandbytes_cpu.dll b/bitsandbytes_windows/libbitsandbytes_cpu.dll
new file mode 100644
index 0000000000000000000000000000000000000000..b733af475eb02eb04f5ad8cbf0530a78a58bc758
Binary files /dev/null and b/bitsandbytes_windows/libbitsandbytes_cpu.dll differ
diff --git a/bitsandbytes_windows/libbitsandbytes_cuda116.dll b/bitsandbytes_windows/libbitsandbytes_cuda116.dll
new file mode 100644
index 0000000000000000000000000000000000000000..84ef81251d1c54b7c62a6856856332811f6c80ba
--- /dev/null
+++ b/bitsandbytes_windows/libbitsandbytes_cuda116.dll
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88f7bd2916ca3effc43f88492f1e1b9088d13cb5be3b4a3a4aede6aa3bf8d412
+size 4724224
diff --git a/bitsandbytes_windows/main.py b/bitsandbytes_windows/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..20a93fa41ad29b0873797a93720fc44fc5714197
--- /dev/null
+++ b/bitsandbytes_windows/main.py
@@ -0,0 +1,166 @@
+"""
+extract factors the build is dependent on:
+[X] compute capability
+    [ ] TODO: Q - What if we have multiple GPUs of different makes?
+- CUDA version
+- Software:
+    - CPU-only: only CPU quantization functions (no optimizer, no matrix multiple)
+    - CuBLAS-LT: full-build 8-bit optimizer
+    - no CuBLAS-LT: no 8-bit matrix multiplication (`nomatmul`)
+
+evaluation:
+    - if paths faulty, return meaningful error
+    - else:
+        - determine CUDA version
+        - determine capabilities
+        - based on that set the default path
+"""
+
+import ctypes
+
+from .paths import determine_cuda_runtime_lib_path
+
+
+def check_cuda_result(cuda, result_val):
+    # 3. Check for CUDA errors
+    if result_val != 0:
+        error_str = ctypes.c_char_p()
+        cuda.cuGetErrorString(result_val, ctypes.byref(error_str))
+        print(f"CUDA exception! Error code: {error_str.value.decode()}")
+
+def get_cuda_version(cuda, cudart_path):
+    # https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART____VERSION.html#group__CUDART____VERSION
+    try:
+        cudart = ctypes.CDLL(cudart_path)
+    except OSError:
+        # TODO: shouldn't we error or at least warn here?
+        print(f'ERROR: libcudart.so could not be read from path: {cudart_path}!')
+        return None
+
+    version = ctypes.c_int()
+    check_cuda_result(cuda, cudart.cudaRuntimeGetVersion(ctypes.byref(version)))
+    version = int(version.value)
+    major = version//1000
+    minor = (version-(major*1000))//10
+
+    if major < 11:
+       print('CUDA SETUP: CUDA version lower than 11 are currently not supported for LLM.int8(). You will be only to use 8-bit optimizers and quantization routines!!')
+
+    return f'{major}{minor}'
+
+
+def get_cuda_lib_handle():
+    # 1. find libcuda.so library (GPU driver) (/usr/lib)
+    try:
+        cuda = ctypes.CDLL("libcuda.so")
+    except OSError:
+        # TODO: shouldn't we error or at least warn here?
+        print('CUDA SETUP: WARNING! libcuda.so not found! Do you have a CUDA driver installed? If you are on a cluster, make sure you are on a CUDA machine!')
+        return None
+    check_cuda_result(cuda, cuda.cuInit(0))
+
+    return cuda
+
+
+def get_compute_capabilities(cuda):
+    """
+    1. find libcuda.so library (GPU driver) (/usr/lib)
+       init_device -> init variables -> call function by reference
+    2. call extern C function to determine CC
+       (https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html)
+    3. Check for CUDA errors
+       https://stackoverflow.com/questions/14038589/what-is-the-canonical-way-to-check-for-errors-using-the-cuda-runtime-api
+    # bits taken from https://gist.github.com/f0k/63a664160d016a491b2cbea15913d549
+    """
+
+
+    nGpus = ctypes.c_int()
+    cc_major = ctypes.c_int()
+    cc_minor = ctypes.c_int()
+
+    device = ctypes.c_int()
+
+    check_cuda_result(cuda, cuda.cuDeviceGetCount(ctypes.byref(nGpus)))
+    ccs = []
+    for i in range(nGpus.value):
+        check_cuda_result(cuda, cuda.cuDeviceGet(ctypes.byref(device), i))
+        ref_major = ctypes.byref(cc_major)
+        ref_minor = ctypes.byref(cc_minor)
+        # 2. call extern C function to determine CC
+        check_cuda_result(
+            cuda, cuda.cuDeviceComputeCapability(ref_major, ref_minor, device)
+        )
+        ccs.append(f"{cc_major.value}.{cc_minor.value}")
+
+    return ccs
+
+
+# def get_compute_capability()-> Union[List[str, ...], None]: # FIXME: error
+def get_compute_capability(cuda):
+    """
+    Extracts the highest compute capbility from all available GPUs, as compute
+    capabilities are downwards compatible. If no GPUs are detected, it returns
+    None.
+    """
+    ccs = get_compute_capabilities(cuda)
+    if ccs is not None:
+        # TODO: handle different compute capabilities; for now, take the max
+        return ccs[-1]
+    return None
+
+
+def evaluate_cuda_setup():
+    print('')
+    print('='*35 + 'BUG REPORT' + '='*35)
+    print('Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues')
+    print('For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link')
+    print('='*80)
+    return "libbitsandbytes_cuda116.dll"            # $$$
+    
+    binary_name = "libbitsandbytes_cpu.so"
+    #if not torch.cuda.is_available():
+        #print('No GPU detected. Loading CPU library...')
+        #return binary_name
+
+    cudart_path = determine_cuda_runtime_lib_path()
+    if cudart_path is None:
+        print(
+            "WARNING: No libcudart.so found! Install CUDA or the cudatoolkit package (anaconda)!"
+        )
+        return binary_name
+
+    print(f"CUDA SETUP: CUDA runtime path found: {cudart_path}")
+    cuda = get_cuda_lib_handle()
+    cc = get_compute_capability(cuda)
+    print(f"CUDA SETUP: Highest compute capability among GPUs detected: {cc}")
+    cuda_version_string = get_cuda_version(cuda, cudart_path)
+
+
+    if cc == '':
+        print(
+            "WARNING: No GPU detected! Check your CUDA paths. Processing to load CPU-only library..."
+        )
+        return binary_name
+
+    # 7.5 is the minimum CC vor cublaslt
+    has_cublaslt = cc in ["7.5", "8.0", "8.6"]
+
+    # TODO:
+    # (1) CUDA missing cases (no CUDA installed by CUDA driver (nvidia-smi accessible)
+    # (2) Multiple CUDA versions installed
+
+    # we use ls -l instead of nvcc to determine the cuda version
+    # since most installations will have the libcudart.so installed, but not the compiler
+    print(f'CUDA SETUP: Detected CUDA version {cuda_version_string}')
+
+    def get_binary_name():
+        "if not has_cublaslt (CC < 7.5), then we have to choose  _nocublaslt.so"
+        bin_base_name = "libbitsandbytes_cuda"
+        if has_cublaslt:
+            return f"{bin_base_name}{cuda_version_string}.so"
+        else:
+            return f"{bin_base_name}{cuda_version_string}_nocublaslt.so"
+
+    binary_name = get_binary_name()
+
+    return binary_name
\ No newline at end of file
diff --git a/config_files/accelerate/default_config.yaml b/config_files/accelerate/default_config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1198fe6d27943b28edb3c312f3ebfcac59ae164c
--- /dev/null
+++ b/config_files/accelerate/default_config.yaml
@@ -0,0 +1,22 @@
+command_file: null
+commands: null
+compute_environment: LOCAL_MACHINE
+deepspeed_config: {}
+distributed_type: 'NO'
+downcast_bf16: 'no'
+dynamo_backend: 'NO'
+fsdp_config: {}
+gpu_ids: all
+machine_rank: 0
+main_process_ip: null
+main_process_port: null
+main_training_function: main
+megatron_lm_config: {}
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+tpu_name: null
+tpu_zone: null
+use_cpu: false
\ No newline at end of file
diff --git a/config_files/accelerate/runpod.yaml b/config_files/accelerate/runpod.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1198fe6d27943b28edb3c312f3ebfcac59ae164c
--- /dev/null
+++ b/config_files/accelerate/runpod.yaml
@@ -0,0 +1,22 @@
+command_file: null
+commands: null
+compute_environment: LOCAL_MACHINE
+deepspeed_config: {}
+distributed_type: 'NO'
+downcast_bf16: 'no'
+dynamo_backend: 'NO'
+fsdp_config: {}
+gpu_ids: all
+machine_rank: 0
+main_process_ip: null
+main_process_port: null
+main_training_function: main
+megatron_lm_config: {}
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+tpu_name: null
+tpu_zone: null
+use_cpu: false
\ No newline at end of file
diff --git a/convert_original_stable_diffusion_to_diffusers.py b/convert_original_stable_diffusion_to_diffusers.py
new file mode 100644
index 0000000000000000000000000000000000000000..de64095523b6cfc0acd25e36b44a03f99885f261
--- /dev/null
+++ b/convert_original_stable_diffusion_to_diffusers.py
@@ -0,0 +1,156 @@
+# coding=utf-8
+# Copyright 2023 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Conversion script for the LDM checkpoints. """
+
+import argparse
+
+import torch
+
+from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_from_original_stable_diffusion_ckpt
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert."
+    )
+    # !wget https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml
+    parser.add_argument(
+        "--original_config_file",
+        default=None,
+        type=str,
+        help="The YAML config file corresponding to the original architecture.",
+    )
+    parser.add_argument(
+        "--num_in_channels",
+        default=None,
+        type=int,
+        help="The number of input channels. If `None` number of input channels will be automatically inferred.",
+    )
+    parser.add_argument(
+        "--scheduler_type",
+        default="pndm",
+        type=str,
+        help="Type of scheduler to use. Should be one of ['pndm', 'lms', 'ddim', 'euler', 'euler-ancestral', 'dpm']",
+    )
+    parser.add_argument(
+        "--pipeline_type",
+        default=None,
+        type=str,
+        help=(
+            "The pipeline type. One of 'FrozenOpenCLIPEmbedder', 'FrozenCLIPEmbedder', 'PaintByExample'"
+            ". If `None` pipeline will be automatically inferred."
+        ),
+    )
+    parser.add_argument(
+        "--image_size",
+        default=None,
+        type=int,
+        help=(
+            "The image size that the model was trained on. Use 512 for Stable Diffusion v1.X and Stable Siffusion v2"
+            " Base. Use 768 for Stable Diffusion v2."
+        ),
+    )
+    parser.add_argument(
+        "--prediction_type",
+        default=None,
+        type=str,
+        help=(
+            "The prediction type that the model was trained on. Use 'epsilon' for Stable Diffusion v1.X and Stable"
+            " Diffusion v2 Base. Use 'v_prediction' for Stable Diffusion v2."
+        ),
+    )
+    parser.add_argument(
+        "--extract_ema",
+        action="store_true",
+        help=(
+            "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights"
+            " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield"
+            " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning."
+        ),
+    )
+    parser.add_argument(
+        "--upcast_attention",
+        action="store_true",
+        help=(
+            "Whether the attention computation should always be upcasted. This is necessary when running stable"
+            " diffusion 2.1."
+        ),
+    )
+    parser.add_argument(
+        "--from_safetensors",
+        action="store_true",
+        help="If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.",
+    )
+    parser.add_argument(
+        "--to_safetensors",
+        action="store_true",
+        help="Whether to store pipeline in safetensors format or not.",
+    )
+    parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.")
+    parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)")
+    parser.add_argument(
+        "--stable_unclip",
+        type=str,
+        default=None,
+        required=False,
+        help="Set if this is a stable unCLIP model. One of 'txt2img' or 'img2img'.",
+    )
+    parser.add_argument(
+        "--stable_unclip_prior",
+        type=str,
+        default=None,
+        required=False,
+        help="Set if this is a stable unCLIP txt2img model. Selects which prior to use. If `--stable_unclip` is set to `txt2img`, the karlo prior (https://huggingface.co/kakaobrain/karlo-v1-alpha/tree/main/prior) is selected by default.",
+    )
+    parser.add_argument(
+        "--clip_stats_path",
+        type=str,
+        help="Path to the clip stats file. Only required if the stable unclip model's config specifies `model.params.noise_aug_config.params.clip_stats_path`.",
+        required=False,
+    )
+    parser.add_argument(
+        "--controlnet", action="store_true", default=None, help="Set flag if this is a controlnet checkpoint."
+    )
+    parser.add_argument("--half", action="store_true", help="Save weights in half precision.")
+    args = parser.parse_args()
+
+    pipe = download_from_original_stable_diffusion_ckpt(
+        checkpoint_path=args.checkpoint_path,
+        original_config_file=args.original_config_file,
+        image_size=args.image_size,
+        prediction_type=args.prediction_type,
+        model_type=args.pipeline_type,
+        extract_ema=args.extract_ema,
+        scheduler_type=args.scheduler_type,
+        num_in_channels=args.num_in_channels,
+        upcast_attention=args.upcast_attention,
+        from_safetensors=args.from_safetensors,
+        device=args.device,
+        stable_unclip=args.stable_unclip,
+        stable_unclip_prior=args.stable_unclip_prior,
+        clip_stats_path=args.clip_stats_path,
+        controlnet=args.controlnet,
+    )
+
+    if args.half:
+        pipe.to(torch_dtype=torch.float16)
+
+    if args.controlnet:
+        # only save the controlnet model
+        pipe.controlnet.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors)
+    else:
+        pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors)
diff --git a/convert_original_stable_diffusion_to_diffusers.py.1 b/convert_original_stable_diffusion_to_diffusers.py.1
new file mode 100644
index 0000000000000000000000000000000000000000..de64095523b6cfc0acd25e36b44a03f99885f261
--- /dev/null
+++ b/convert_original_stable_diffusion_to_diffusers.py.1
@@ -0,0 +1,156 @@
+# coding=utf-8
+# Copyright 2023 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Conversion script for the LDM checkpoints. """
+
+import argparse
+
+import torch
+
+from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_from_original_stable_diffusion_ckpt
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert."
+    )
+    # !wget https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml
+    parser.add_argument(
+        "--original_config_file",
+        default=None,
+        type=str,
+        help="The YAML config file corresponding to the original architecture.",
+    )
+    parser.add_argument(
+        "--num_in_channels",
+        default=None,
+        type=int,
+        help="The number of input channels. If `None` number of input channels will be automatically inferred.",
+    )
+    parser.add_argument(
+        "--scheduler_type",
+        default="pndm",
+        type=str,
+        help="Type of scheduler to use. Should be one of ['pndm', 'lms', 'ddim', 'euler', 'euler-ancestral', 'dpm']",
+    )
+    parser.add_argument(
+        "--pipeline_type",
+        default=None,
+        type=str,
+        help=(
+            "The pipeline type. One of 'FrozenOpenCLIPEmbedder', 'FrozenCLIPEmbedder', 'PaintByExample'"
+            ". If `None` pipeline will be automatically inferred."
+        ),
+    )
+    parser.add_argument(
+        "--image_size",
+        default=None,
+        type=int,
+        help=(
+            "The image size that the model was trained on. Use 512 for Stable Diffusion v1.X and Stable Siffusion v2"
+            " Base. Use 768 for Stable Diffusion v2."
+        ),
+    )
+    parser.add_argument(
+        "--prediction_type",
+        default=None,
+        type=str,
+        help=(
+            "The prediction type that the model was trained on. Use 'epsilon' for Stable Diffusion v1.X and Stable"
+            " Diffusion v2 Base. Use 'v_prediction' for Stable Diffusion v2."
+        ),
+    )
+    parser.add_argument(
+        "--extract_ema",
+        action="store_true",
+        help=(
+            "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights"
+            " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield"
+            " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning."
+        ),
+    )
+    parser.add_argument(
+        "--upcast_attention",
+        action="store_true",
+        help=(
+            "Whether the attention computation should always be upcasted. This is necessary when running stable"
+            " diffusion 2.1."
+        ),
+    )
+    parser.add_argument(
+        "--from_safetensors",
+        action="store_true",
+        help="If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.",
+    )
+    parser.add_argument(
+        "--to_safetensors",
+        action="store_true",
+        help="Whether to store pipeline in safetensors format or not.",
+    )
+    parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.")
+    parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)")
+    parser.add_argument(
+        "--stable_unclip",
+        type=str,
+        default=None,
+        required=False,
+        help="Set if this is a stable unCLIP model. One of 'txt2img' or 'img2img'.",
+    )
+    parser.add_argument(
+        "--stable_unclip_prior",
+        type=str,
+        default=None,
+        required=False,
+        help="Set if this is a stable unCLIP txt2img model. Selects which prior to use. If `--stable_unclip` is set to `txt2img`, the karlo prior (https://huggingface.co/kakaobrain/karlo-v1-alpha/tree/main/prior) is selected by default.",
+    )
+    parser.add_argument(
+        "--clip_stats_path",
+        type=str,
+        help="Path to the clip stats file. Only required if the stable unclip model's config specifies `model.params.noise_aug_config.params.clip_stats_path`.",
+        required=False,
+    )
+    parser.add_argument(
+        "--controlnet", action="store_true", default=None, help="Set flag if this is a controlnet checkpoint."
+    )
+    parser.add_argument("--half", action="store_true", help="Save weights in half precision.")
+    args = parser.parse_args()
+
+    pipe = download_from_original_stable_diffusion_ckpt(
+        checkpoint_path=args.checkpoint_path,
+        original_config_file=args.original_config_file,
+        image_size=args.image_size,
+        prediction_type=args.prediction_type,
+        model_type=args.pipeline_type,
+        extract_ema=args.extract_ema,
+        scheduler_type=args.scheduler_type,
+        num_in_channels=args.num_in_channels,
+        upcast_attention=args.upcast_attention,
+        from_safetensors=args.from_safetensors,
+        device=args.device,
+        stable_unclip=args.stable_unclip,
+        stable_unclip_prior=args.stable_unclip_prior,
+        clip_stats_path=args.clip_stats_path,
+        controlnet=args.controlnet,
+    )
+
+    if args.half:
+        pipe.to(torch_dtype=torch.float16)
+
+    if args.controlnet:
+        # only save the controlnet model
+        pipe.controlnet.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors)
+    else:
+        pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors)
diff --git a/convert_original_stable_diffusion_to_diffusers.py.2 b/convert_original_stable_diffusion_to_diffusers.py.2
new file mode 100644
index 0000000000000000000000000000000000000000..de64095523b6cfc0acd25e36b44a03f99885f261
--- /dev/null
+++ b/convert_original_stable_diffusion_to_diffusers.py.2
@@ -0,0 +1,156 @@
+# coding=utf-8
+# Copyright 2023 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Conversion script for the LDM checkpoints. """
+
+import argparse
+
+import torch
+
+from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_from_original_stable_diffusion_ckpt
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert."
+    )
+    # !wget https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml
+    parser.add_argument(
+        "--original_config_file",
+        default=None,
+        type=str,
+        help="The YAML config file corresponding to the original architecture.",
+    )
+    parser.add_argument(
+        "--num_in_channels",
+        default=None,
+        type=int,
+        help="The number of input channels. If `None` number of input channels will be automatically inferred.",
+    )
+    parser.add_argument(
+        "--scheduler_type",
+        default="pndm",
+        type=str,
+        help="Type of scheduler to use. Should be one of ['pndm', 'lms', 'ddim', 'euler', 'euler-ancestral', 'dpm']",
+    )
+    parser.add_argument(
+        "--pipeline_type",
+        default=None,
+        type=str,
+        help=(
+            "The pipeline type. One of 'FrozenOpenCLIPEmbedder', 'FrozenCLIPEmbedder', 'PaintByExample'"
+            ". If `None` pipeline will be automatically inferred."
+        ),
+    )
+    parser.add_argument(
+        "--image_size",
+        default=None,
+        type=int,
+        help=(
+            "The image size that the model was trained on. Use 512 for Stable Diffusion v1.X and Stable Siffusion v2"
+            " Base. Use 768 for Stable Diffusion v2."
+        ),
+    )
+    parser.add_argument(
+        "--prediction_type",
+        default=None,
+        type=str,
+        help=(
+            "The prediction type that the model was trained on. Use 'epsilon' for Stable Diffusion v1.X and Stable"
+            " Diffusion v2 Base. Use 'v_prediction' for Stable Diffusion v2."
+        ),
+    )
+    parser.add_argument(
+        "--extract_ema",
+        action="store_true",
+        help=(
+            "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights"
+            " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield"
+            " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning."
+        ),
+    )
+    parser.add_argument(
+        "--upcast_attention",
+        action="store_true",
+        help=(
+            "Whether the attention computation should always be upcasted. This is necessary when running stable"
+            " diffusion 2.1."
+        ),
+    )
+    parser.add_argument(
+        "--from_safetensors",
+        action="store_true",
+        help="If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.",
+    )
+    parser.add_argument(
+        "--to_safetensors",
+        action="store_true",
+        help="Whether to store pipeline in safetensors format or not.",
+    )
+    parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.")
+    parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)")
+    parser.add_argument(
+        "--stable_unclip",
+        type=str,
+        default=None,
+        required=False,
+        help="Set if this is a stable unCLIP model. One of 'txt2img' or 'img2img'.",
+    )
+    parser.add_argument(
+        "--stable_unclip_prior",
+        type=str,
+        default=None,
+        required=False,
+        help="Set if this is a stable unCLIP txt2img model. Selects which prior to use. If `--stable_unclip` is set to `txt2img`, the karlo prior (https://huggingface.co/kakaobrain/karlo-v1-alpha/tree/main/prior) is selected by default.",
+    )
+    parser.add_argument(
+        "--clip_stats_path",
+        type=str,
+        help="Path to the clip stats file. Only required if the stable unclip model's config specifies `model.params.noise_aug_config.params.clip_stats_path`.",
+        required=False,
+    )
+    parser.add_argument(
+        "--controlnet", action="store_true", default=None, help="Set flag if this is a controlnet checkpoint."
+    )
+    parser.add_argument("--half", action="store_true", help="Save weights in half precision.")
+    args = parser.parse_args()
+
+    pipe = download_from_original_stable_diffusion_ckpt(
+        checkpoint_path=args.checkpoint_path,
+        original_config_file=args.original_config_file,
+        image_size=args.image_size,
+        prediction_type=args.prediction_type,
+        model_type=args.pipeline_type,
+        extract_ema=args.extract_ema,
+        scheduler_type=args.scheduler_type,
+        num_in_channels=args.num_in_channels,
+        upcast_attention=args.upcast_attention,
+        from_safetensors=args.from_safetensors,
+        device=args.device,
+        stable_unclip=args.stable_unclip,
+        stable_unclip_prior=args.stable_unclip_prior,
+        clip_stats_path=args.clip_stats_path,
+        controlnet=args.controlnet,
+    )
+
+    if args.half:
+        pipe.to(torch_dtype=torch.float16)
+
+    if args.controlnet:
+        # only save the controlnet model
+        pipe.controlnet.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors)
+    else:
+        pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors)
diff --git a/dataset/.gitkeep b/dataset/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/docker-compose.yaml b/docker-compose.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..73661d3090cc318469c448b403a15a1e7e4aa927
--- /dev/null
+++ b/docker-compose.yaml
@@ -0,0 +1,33 @@
+version: "3.8"
+services:
+  kohya-ss-gui:
+    container_name: kohya-ss-gui
+    image: kohya-ss-gui:latest
+    build:
+      context: .
+    ports:
+      - 127.0.0.1:3000:3000
+      - 7860:7860
+      - 6006:6006
+    tty: true
+    ipc: host
+    environment:
+      CLI_ARGS: ""
+      SAFETENSORS_FAST_GPU: 1
+      DISPLAY: $DISPLAY
+    tmpfs:
+      - /tmp      
+    volumes:
+      - ./dataset:/dataset
+      - ./.cache/user:/home/appuser/.cache
+      - ./.cache/triton:/home/appuser/.triton    
+      - ./.cache/config:/app/appuser/.config
+      - ./.cache/nv:/home/appuser/.nv 
+      - ./.cache/keras:/home/appuser/.keras      
+      - /tmp/.X11-unix:/tmp/.X11-unix
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              device_ids: ['all']
diff --git a/docs/Finetuning/top_level.md b/docs/Finetuning/top_level.md
new file mode 100644
index 0000000000000000000000000000000000000000..5530cd3e93d98a081cabf7edb23bf7f1eaec2860
--- /dev/null
+++ b/docs/Finetuning/top_level.md
@@ -0,0 +1,28 @@
+# Finetuning Resource Guide
+
+This guide is a resource compilation to facilitate the development of robust LoRA models.
+
+-Need to add resources here
+
+## Guidelines for SDXL Finetuning 
+
+- Set the `Max resolution` to at least 1024x1024, as this is the standard resolution for SDXL.
+- The fine-tuning can be done with 24GB GPU memory with the batch size of 1.
+  - Train U-Net only.
+  - Use gradient checkpointing.
+  - Use `--cache_text_encoder_outputs` option and caching latents.
+  - Use Adafactor optimizer. RMSprop 8bit or Adagrad 8bit may work. AdamW 8bit doesn't seem to work.
+- PyTorch 2 seems to use slightly less GPU memory than PyTorch 1.
+
+Example of the optimizer settings for Adafactor with the fixed learning rate:
+```
+optimizer_type = "adafactor"
+optimizer_args = [ "scale_parameter=False", "relative_step=False", "warmup_init=False" ]
+lr_scheduler = "constant_with_warmup"
+lr_warmup_steps = 100
+learning_rate = 4e-7 # SDXL original learning rate
+```
+
+## Resource Contributions
+
+If you have valuable resources to add, kindly create a PR on Github.
\ No newline at end of file
diff --git a/docs/LoRA/top_level.md b/docs/LoRA/top_level.md
new file mode 100644
index 0000000000000000000000000000000000000000..d2c3c6d3ab42f9ebd389e5123a6caa855a670cd4
--- /dev/null
+++ b/docs/LoRA/top_level.md
@@ -0,0 +1,26 @@
+# LoRA Resource Guide
+
+This guide is a resource compilation to facilitate the development of robust LoRA models.
+
+Access EDG's tutorials here: https://ko-fi.com/post/EDGs-tutorials-P5P6KT5MT
+
+## Guidelines for SDXL LoRA Training 
+
+- Set the `Max resolution` to at least 1024x1024, as this is the standard resolution for SDXL.
+- Use a GPU that has at least 12GB memory for the LoRA training process.
+- We strongly recommend using the `--train_unet_only` option for SDXL LoRA to avoid unforeseen training results caused by dual text encoders in SDXL.
+- PyTorch 2 tends to use less GPU memory than PyTorch 1.
+
+Here's an example configuration for the Adafactor optimizer with a fixed learning rate:
+
+```
+optimizer_type = "adafactor"
+optimizer_args = [ "scale_parameter=False", "relative_step=False", "warmup_init=False" ]
+lr_scheduler = "constant_with_warmup"
+lr_warmup_steps = 100
+learning_rate = 4e-7 # This is the standard learning rate for SDXL
+```
+
+## Resource Contributions
+
+If you have valuable resources to add, kindly create a PR on Github.
\ No newline at end of file
diff --git a/docs/config_README-ja.md b/docs/config_README-ja.md
new file mode 100644
index 0000000000000000000000000000000000000000..7f2b6c4c1e3859a6ce79a4b6ece2174b430d1d20
--- /dev/null
+++ b/docs/config_README-ja.md
@@ -0,0 +1,279 @@
+For non-Japanese speakers: this README is provided only in Japanese in the current state. Sorry for inconvenience. We will provide English version in the near future.
+
+`--dataset_config` で渡すことができる設定ファイルに関する説明です。
+
+## 概要
+
+設定ファイルを渡すことにより、ユーザが細かい設定を行えるようにします。
+
+* 複数のデータセットが設定可能になります
+    * 例えば `resolution` をデータセットごとに設定して、それらを混合して学習できます。
+    * DreamBooth の手法と fine tuning の手法の両方に対応している学習方法では、DreamBooth 方式と fine tuning 方式のデータセットを混合することが可能です。
+* サブセットごとに設定を変更することが可能になります
+    * データセットを画像ディレクトリ別またはメタデータ別に分割したものがサブセットです。いくつかのサブセットが集まってデータセットを構成します。
+    * `keep_tokens` や `flip_aug` 等のオプションはサブセットごとに設定可能です。一方、`resolution` や `batch_size` といったオプションはデータセットごとに設定可能で、同じデータセットに属するサブセットでは値が共通になります。詳しくは後述します。
+
+設定ファイルの形式は JSON か TOML を利用できます。記述のしやすさを考えると [TOML](https://toml.io/ja/v1.0.0-rc.2) を利用するのがオススメです。以下、TOML の利用を前提に説明します。
+
+TOML で記述した設定ファイルの例です。
+
+```toml
+[general]
+shuffle_caption = true
+caption_extension = '.txt'
+keep_tokens = 1
+
+# これは DreamBooth 方式のデータセット
+[[datasets]]
+resolution = 512
+batch_size = 4
+keep_tokens = 2
+
+  [[datasets.subsets]]
+  image_dir = 'C:\hoge'
+  class_tokens = 'hoge girl'
+  # このサブセットは keep_tokens = 2 （所属する datasets の値が使われる）
+
+  [[datasets.subsets]]
+  image_dir = 'C:\fuga'
+  class_tokens = 'fuga boy'
+  keep_tokens = 3
+
+  [[datasets.subsets]]
+  is_reg = true
+  image_dir = 'C:\reg'
+  class_tokens = 'human'
+  keep_tokens = 1
+
+# これは fine tuning 方式のデータセット
+[[datasets]]
+resolution = [768, 768]
+batch_size = 2
+
+  [[datasets.subsets]]
+  image_dir = 'C:\piyo'
+  metadata_file = 'C:\piyo\piyo_md.json'
+  # このサブセットは keep_tokens = 1 （general の値が使われる）
+```
+
+この例では、3 つのディレクトリを DreamBooth 方式のデータセットとして 512x512 (batch size 4) で学習させ、1 つのディレクトリを fine tuning 方式のデータセットとして 768x768 (batch size 2) で学習させることになります。
+
+## データセット・サブセットに関する設定
+
+データセット・サブセットに関する設定は、登録可能な箇所がいくつかに分かれています。
+
+* `[general]`
+    * 全データセットまたは全サブセットに適用されるオプションを指定する箇所です。
+    * データセットごとの設定及びサブセットごとの設定に同名のオプションが存在していた場合には、データセット・サブセットごとの設定が優先されます。
+* `[[datasets]]`
+    * `datasets` はデータセットに関する設定の登録箇所になります。各データセットに個別に適用されるオプションを指定する箇所です。
+    * サブセットごとの設定が存在していた場合には、サブセットごとの設定が優先されます。
+* `[[datasets.subsets]]`
+    * `datasets.subsets` はサブセットに関する設定の登録箇所になります。各サブセットに個別に適用されるオプションを指定する箇所です。
+
+先程の例における、画像ディレクトリと登録箇所の対応に関するイメージ図です。
+
+```
+C:\
+├─ hoge  ->  [[datasets.subsets]] No.1  ┐                        ┐
+├─ fuga  ->  [[datasets.subsets]] No.2  |->  [[datasets]] No.1   |->  [general]
+├─ reg   ->  [[datasets.subsets]] No.3  ┘                        |
+└─ piyo  ->  [[datasets.subsets]] No.4  -->  [[datasets]] No.2   ┘
+```
+
+画像ディレクトリがそれぞれ1つの `[[datasets.subsets]]` に対応しています。そして `[[datasets.subsets]]` が1つ以上組み合わさって1つの `[[datasets]]` を構成します。`[general]` には全ての `[[datasets]]`, `[[datasets.subsets]]` が属します。
+
+登録箇所ごとに指定可能なオプションは異なりますが、同名のオプションが指定された場合は下位の登録箇所にある値が優先されます。先程の例の `keep_tokens` オプションの扱われ方を確認してもらうと理解しやすいかと思います。
+
+加えて、学習方法が対応している手法によっても指定可能なオプションが変化します。
+
+* DreamBooth 方式専用のオプション
+* fine tuning 方式専用のオプション
+* caption dropout の手法が使える場合のオプション
+
+DreamBooth の手法と fine tuning の手法の両方とも利用可能な学習方法では、両者を併用することができます。
+併用する際の注意点として、DreamBooth 方式なのか fine tuning 方式なのかはデータセット単位で判別を行っているため、同じデータセット中に DreamBooth 方式のサブセットと fine tuning 方式のサブセットを混在させることはできません。
+つまり、これらを併用したい場合には異なる方式のサブセットが異なるデータセットに所属するように設定する必要があります。
+
+プログラムの挙動としては、後述する `metadata_file` オプションが存在していたら fine tuning 方式のサブセットだと判断します。
+そのため、同一のデータセットに所属するサブセットについて言うと、「全てが `metadata_file` オプションを持つ」か「全てが `metadata_file` オプションを持たない」かのどちらかになっていれば問題ありません。
+
+以下、利用可能なオプションを説明します。コマンドライン引数と名称が同一のオプションについては、基本的に説明を割愛します。他の README を参照してください。
+
+### 全学習方法で共通のオプション
+
+学習方法によらずに指定可能なオプションです。
+
+#### データセット向けオプション
+
+データセットの設定に関わるオプションです。`datasets.subsets` には記述できません。
+
+| オプション名 | 設定例 | `[general]` | `[[datasets]]` |
+| ---- | ---- | ---- | ---- |
+| `batch_size` | `1` | o | o |
+| `bucket_no_upscale` | `true` | o | o |
+| `bucket_reso_steps` | `64` | o | o |
+| `enable_bucket` | `true` | o | o |
+| `max_bucket_reso` | `1024` | o | o |
+| `min_bucket_reso` | `128` | o | o |
+| `resolution` | `256`, `[512, 512]` | o | o |
+
+* `batch_size`
+    * コマンドライン引数の `--train_batch_size` と同等です。
+
+これらの設定はデータセットごとに固定です。
+つまり、データセットに所属するサブセットはこれらの設定を共有することになります。
+例えば解像度が異なるデータセットを用意したい場合は、上に挙げた例のように別々のデータセットとして定義すれば別々の解像度を設定可能です。
+
+#### サブセット向けオプション
+
+サブセットの設定に関わるオプションです。
+
+| オプション名 | 設定例 | `[general]` | `[[datasets]]` | `[[dataset.subsets]]` |
+| ---- | ---- | ---- | ---- | ---- |
+| `color_aug` | `false` | o | o | o |
+| `face_crop_aug_range` | `[1.0, 3.0]` | o | o | o |
+| `flip_aug` | `true` | o | o | o |
+| `keep_tokens` | `2` | o | o | o |
+| `num_repeats` | `10` | o | o | o |
+| `random_crop` | `false` | o | o | o |
+| `shuffle_caption` | `true` | o | o | o |
+
+* `num_repeats`
+    * サブセットの画像の繰り返し回数を指定します。fine tuning における `--dataset_repeats` に相当しますが、`num_repeats` はどの学習方法でも指定可能です。
+
+### DreamBooth 方式専用のオプション
+
+DreamBooth 方式のオプションは、サブセット向けオプションのみ存在します。
+
+#### サブセット向けオプション
+
+DreamBooth 方式のサブセットの設定に関わるオプションです。
+
+| オプション名 | 設定例 | `[general]` | `[[datasets]]` | `[[dataset.subsets]]` |
+| ---- | ---- | ---- | ---- | ---- |
+| `image_dir` | `‘C:\hoge’` | - | - | o（必須） |
+| `caption_extension` | `".txt"` | o | o | o |
+| `class_tokens` | `“sks girl”` | - | - | o |
+| `is_reg` | `false` | - | - | o |
+
+まず注意点として、 `image_dir` には画像ファイルが直下に置かれているパスを指定する必要があります。従来の DreamBooth の手法ではサブディレクトリに画像を置く必要がありましたが、そちらとは仕様に互換性がありません。また、`5_cat` のようなフォルダ名にしても、画像の繰り返し回数とクラス名は反映されません。これらを個別に設定したい場合、`num_repeats` と `class_tokens` で明示的に指定する必要があることに注意してください。
+
+* `image_dir`
+    * 画像ディレクトリのパスを指定します。指定必須オプションです。
+    * 画像はディレクトリ直下に置かれている必要があります。
+* `class_tokens`
+    * クラストークンを設定します。
+    * 画像に対応する caption ファイルが存在しない場合にのみ学習時に利用されます。利用するかどうかの判定は画像ごとに行います。`class_tokens` を指定しなかった場合に caption ファイルも見つからなかった場合にはエラーになります。
+* `is_reg`
+    * サブセットの画像が正規化用かどうかを指定します。指定しなかった場合は `false` として、つまり正規化画像ではないとして扱います。
+
+### fine tuning 方式専用のオプション
+
+fine tuning 方式のオプションは、サブセット向けオプションのみ存在します。
+
+#### サブセット向けオプション
+
+fine tuning 方式のサブセットの設定に関わるオプションです。
+
+| オプション名 | 設定例 | `[general]` | `[[datasets]]` | `[[dataset.subsets]]` |
+| ---- | ---- | ---- | ---- | ---- |
+| `image_dir` | `‘C:\hoge’` | - | - | o |
+| `metadata_file` | `'C:\piyo\piyo_md.json'` | - | - | o（必須） |
+
+* `image_dir`
+    * 画像ディレクトリのパスを指定します。DreamBooth の手法の方とは異なり指定は必須ではありませんが、設定することを推奨します。
+        * 指定する必要がない状況としては、メタデータファイルの生成時に `--full_path` を付与して実行していた場合です。
+    * 画像はディレクトリ直下に置かれている必要があります。
+* `metadata_file`
+    * サブセットで利用されるメタデータファイルのパスを指定します。指定必須オプションです。
+        * コマンドライン引数の `--in_json` と同等です。
+    * サブセットごとにメタデータファイルを指定する必要がある仕様上、ディレクトリを跨いだメタデータを1つのメタデータファイルとして作成することは避けた方が良いでしょう。画像ディレクトリごとにメタデータファイルを用意し、それらを別々のサブセットとして登録することを強く推奨します。
+
+### caption dropout の手法が使える場合に指定可能なオプション
+
+caption dropout の手法が使える場合のオプションは、サブセット向けオプションのみ存在します。
+DreamBooth 方式か fine tuning 方式かに関わらず、caption dropout に対応している学習方法であれば指定可能です。
+
+#### サブセット向けオプション
+
+caption dropout が使えるサブセットの設定に関わるオプションです。
+
+| オプション名 | `[general]` | `[[datasets]]` | `[[dataset.subsets]]` |
+| ---- | ---- | ---- | ---- |
+| `caption_dropout_every_n_epochs` | o | o | o |
+| `caption_dropout_rate` | o | o | o |
+| `caption_tag_dropout_rate` | o | o | o |
+
+## 重複したサブセットが存在する時の挙動
+
+DreamBooth 方式のデータセットの場合、その中にある `image_dir` が同一のサブセットは重複していると見なされます。
+fine tuning 方式のデータセットの場合は、その中にある `metadata_file` が同一のサブセットは重複していると見なされます。
+データセット中に重複したサブセットが存在する場合、2個目以降は無視されます。
+
+一方、異なるデータセットに所属している場合は、重複しているとは見なされません。
+例えば、以下のように同一の `image_dir` を持つサブセットを別々のデータセットに入れた場合には、重複していないと見なします。
+これは、同じ画像でも異なる解像度で学習したい場合に役立ちます。
+
+```toml
+# 別々のデータセットに存在している場合は重複とは見なされず、両方とも学習に使われる
+
+[[datasets]]
+resolution = 512
+
+  [[datasets.subsets]]
+  image_dir = 'C:\hoge'
+
+[[datasets]]
+resolution = 768
+
+  [[datasets.subsets]]
+  image_dir = 'C:\hoge'
+```
+
+## コマンドライン引数との併用
+
+設定ファイルのオプションの中には、コマンドライン引数のオプションと役割が重複しているものがあります。
+
+以下に挙げるコマンドライン引数のオプションは、設定ファイルを渡した場合には無視されます。
+
+* `--train_data_dir`
+* `--reg_data_dir`
+* `--in_json`
+
+以下に挙げるコマンドライン引数のオプションは、コマンドライン引数と設定ファイルで同時に指定された場合、コマンドライン引数の値よりも設定ファイルの値が優先されます。特に断りがなければ同名のオプションとなります。
+
+| コマンドライン引数のオプション     | 優先される設定ファイルのオプション |
+| ---------------------------------- | ---------------------------------- |
+| `--bucket_no_upscale`              |                                    |
+| `--bucket_reso_steps`              |                                    |
+| `--caption_dropout_every_n_epochs` |                                    |
+| `--caption_dropout_rate`           |                                    |
+| `--caption_extension`              |                                    |
+| `--caption_tag_dropout_rate`       |                                    |
+| `--color_aug`                      |                                    |
+| `--dataset_repeats`                | `num_repeats`                      |
+| `--enable_bucket`                  |                                    |
+| `--face_crop_aug_range`            |                                    |
+| `--flip_aug`                       |                                    |
+| `--keep_tokens`                    |                                    |
+| `--min_bucket_reso`                |                                    |
+| `--random_crop`                    |                                    |
+| `--resolution`                     |                                    |
+| `--shuffle_caption`                |                                    |
+| `--train_batch_size`               | `batch_size`                       |
+
+## エラーの手引き
+
+現在、外部ライブラリを利用して設定ファイルの記述が正しいかどうかをチェックしているのですが、整備が行き届いておらずエラーメッセージがわかりづらいという問題があります。
+将来的にはこの問題の改善に取り組む予定です。
+
+次善策として、頻出のエラーとその対処法について載せておきます。
+正しいはずなのにエラーが出る場合、エラー内容がどうしても分からない場合は、バグかもしれないのでご連絡ください。
+
+* `voluptuous.error.MultipleInvalid: required key not provided @ ...`: 指定必須のオプションが指定されていないというエラーです。指定を忘れているか、オプション名を間違って記述している可能性が高いです。
+  * `...` の箇所にはエラーが発生した場所が載っています。例えば `voluptuous.error.MultipleInvalid: required key not provided @ data['datasets'][0]['subsets'][0]['image_dir']` のようなエラーが出たら、0 番目の `datasets` 中の 0 番目の `subsets` の設定に `image_dir` が存在しないということになります。
+* `voluptuous.error.MultipleInvalid: expected int for dictionary value @ ...`: 指定する値の形式が不正というエラーです。値の形式が間違っている可能性が高いです。`int` の部分は対象となるオプションによって変わります。この README に載っているオプションの「設定例」が役立つかもしれません。
+* `voluptuous.error.MultipleInvalid: extra keys not allowed @ ...`: 対応していないオプション名が存在している場合に発生するエラーです。オプション名を間違って記述しているか、誤って紛れ込んでいる可能性が高いです。
+
+
diff --git a/docs/fine_tune_README_ja.md b/docs/fine_tune_README_ja.md
new file mode 100644
index 0000000000000000000000000000000000000000..686947c952b19c016974792cdc5f4f903701cfc9
--- /dev/null
+++ b/docs/fine_tune_README_ja.md
@@ -0,0 +1,140 @@
+NovelAIの提案した学習手法、自動キャプションニング、タグ付け、Windows＋VRAM 12GB（SD v1.xの場合）環境等に対応したfine tuningです。ここでfine tuningとは、モデルを画像とキャプションで学習することを指します（LoRAやTextual Inversion、Hypernetworksは含みません）
+
+[学習についての共通ドキュメント](./train_README-ja.md) もあわせてご覧ください。
+
+# 概要
+
+Diffusersを用いてStable DiffusionのU-Netのfine tuningを行います。NovelAIの記事にある以下の改善に対応しています（Aspect Ratio BucketingについてはNovelAIのコードを参考にしましたが、最終的なコードはすべてオリジナルです）。
+
+* CLIP（Text Encoder）の最後の層ではなく最後から二番目の層の出力を用いる。
+* 正方形以外の解像度での学習（Aspect Ratio Bucketing） 。
+* トークン長を75から225に拡張する。
+* BLIPによるキャプショニング（キャプションの自動作成）、DeepDanbooruまたはWD14Taggerによる自動タグ付けを行う。
+* Hypernetworkの学習にも対応する。
+* Stable Diffusion v2.0（baseおよび768/v）に対応。
+* VAEの出力をあらかじめ取得しディスクに保存しておくことで、学習の省メモリ化、高速化を図る。
+
+デフォルトではText Encoderの学習は行いません。モデル全体のfine tuningではU-Netだけを学習するのが一般的なようです（NovelAIもそのようです）。オプション指定でText Encoderも学習対象とできます。
+
+# 追加機能について
+
+## CLIPの出力の変更
+
+プロンプトを画像に反映するため、テキストの特徴量への変換を行うのがCLIP（Text Encoder）です。Stable DiffusionではCLIPの最後の層の出力を用いていますが、それを最後から二番目の層の出力を用いるよう変更できます。NovelAIによると、これによりより正確にプロンプトが反映されるようになるとのことです。
+元のまま、最後の層の出力を用いることも可能です。
+
+※Stable Diffusion 2.0では最後から二番目の層をデフォルトで使います。clip_skipオプションを指定しないでください。
+
+## 正方形以外の解像度での学習
+
+Stable Diffusionは512\*512で学習されていますが、それに加えて256\*1024や384\*640といった解像度でも学習します。これによりトリミングされる部分が減り、より正しくプロンプトと画像の関係が学習されることが期待されます。
+学習解像度はパラメータとして与えられた解像度の面積（＝メモリ使用量）を超えない範囲で、64ピクセル単位で縦横に調整、作成されます。
+
+機械学習では入力サイズをすべて統一するのが一般的ですが、特に制約があるわけではなく、実際は同一のバッチ内で統一されていれば大丈夫です。NovelAIの言うbucketingは、あらかじめ教師データを、アスペクト比に応じた学習解像度ごとに分類しておくことを指しているようです。そしてバッチを各bucket内の画像で作成することで、バッチの画像サイズを統一します。
+
+## トークン長の75から225への拡張
+
+Stable Diffusionでは最大75トークン（開始・終了を含むと77トークン）ですが、それを225トークンまで拡張します。
+ただしCLIPが受け付ける最大長は75トークンですので、225トークンの場合、単純に三分割してCLIPを呼び出してから結果を連結しています。
+
+※これが望ましい実装なのかどうかはいまひとつわかりません。とりあえず動いてはいるようです。特に2.0では何も参考になる実装がないので独自に実装してあります。
+
+※Automatic1111氏のWeb UIではカンマを意識して分割、といったこともしているようですが、私の場合はそこまでしておらず単純な分割です。
+
+# 学習の手順
+
+あらかじめこのリポジトリのREADMEを参照し、環境整備を行ってください。
+
+## データの準備
+
+[学習データの準備について](./train_README-ja.md) を参照してください。fine tuningではメタデータを用いるfine tuning方式のみ対応しています。
+
+## 学習の実行
+たとえば以下のように実行します。以下は省メモリ化のための設定です。それぞれの行を必要に応じて書き換えてください。
+
+```
+accelerate launch --num_cpu_threads_per_process 1 fine_tune.py 
+    --pretrained_model_name_or_path=<.ckptまたは.safetensordまたはDiffusers版モデルのディレクトリ> 
+    --output_dir=<学習したモデルの出力先フォルダ>  
+    --output_name=<学習したモデル出力時のファイル名> 
+    --dataset_config=<データ準備で作成した.tomlファイル> 
+    --save_model_as=safetensors 
+    --learning_rate=5e-6 --max_train_steps=10000 
+    --use_8bit_adam --xformers --gradient_checkpointing
+    --mixed_precision=fp16
+```
+
+`num_cpu_threads_per_process` には通常は1を指定するとよいようです。
+
+`pretrained_model_name_or_path` に追加学習を行う元となるモデルを指定します。Stable Diffusionのcheckpointファイル（.ckptまたは.safetensors）、Diffusersのローカルディスクにあるモデルディレクトリ、DiffusersのモデルID（"stabilityai/stable-diffusion-2"など）が指定できます。
+
+`output_dir` に学習後のモデルを保存するフォルダを指定します。`output_name` にモデルのファイル名を拡張子を除いて指定します。`save_model_as` でsafetensors形式での保存を指定しています。
+
+`dataset_config` に `.toml` ファイルを指定します。ファイル内でのバッチサイズ指定は、当初はメモリ消費を抑えるために `1` としてください。
+
+学習させるステップ数 `max_train_steps` を10000とします。学習率 `learning_rate` はここでは5e-6を指定しています。
+
+省メモリ化のため `mixed_precision="fp16"` を指定します（RTX30 シリーズ以降では `bf16` も指定できます。環境整備時にaccelerateに行った設定と合わせてください）。また `gradient_checkpointing` を指定します。
+
+オプティマイザ（モデルを学習データにあうように最適化＝学習させるクラス）にメモリ消費の少ない 8bit AdamW を使うため、 `optimizer_type="AdamW8bit"` を指定します。
+
+`xformers` オプションを指定し、xformersのCrossAttentionを用います。xformersをインストールしていない場合やエラーとなる場合（環境にもよりますが `mixed_precision="no"` の場合など）、代わりに `mem_eff_attn` オプションを指定すると省メモリ版CrossAttentionを使用します（速度は遅くなります）。
+
+ある程度メモリがある場合は、`.toml` ファイルを編集してバッチサイズをたとえば `4` くらいに増やしてください（高速化と精度向上の可能性があります）。
+
+### よく使われるオプションについて
+
+以下の場合にはオプションに関するドキュメントを参照してください。
+
+- Stable Diffusion 2.xまたはそこからの派生モデルを学習する
+- clip skipを2以上を前提としたモデルを学習する
+- 75トークンを超えたキャプションで学習する
+
+### バッチサイズについて
+
+モデル全体を学習するためLoRA等の学習に比べるとメモリ消費量は多くなります（DreamBoothと同じ）。
+
+### 学習率について
+
+1e-6から5e-6程度が一般的なようです。他のfine tuningの例なども参照してみてください。
+
+### 以前の形式のデータセット指定をした場合のコマンドライン
+
+解像度やバッチサイズをオプションで指定します。コマンドラインの例は以下の通りです。
+
+```
+accelerate launch --num_cpu_threads_per_process 1 fine_tune.py 
+    --pretrained_model_name_or_path=model.ckpt 
+    --in_json meta_lat.json 
+    --train_data_dir=train_data 
+    --output_dir=fine_tuned 
+    --shuffle_caption 
+    --train_batch_size=1 --learning_rate=5e-6 --max_train_steps=10000 
+    --use_8bit_adam --xformers --gradient_checkpointing
+    --mixed_precision=bf16
+    --save_every_n_epochs=4
+```
+
+<!-- 
+### 勾配をfp16とした学習（実験的機能）
+full_fp16オプションを指定すると勾配を通常のfloat32からfloat16（fp16）に変更して学習します（mixed precisionではなく完全なfp16学習になるようです）。これによりSD1.xの512*512サイズでは8GB未満、SD2.xの512*512サイズで12GB未満のVRAM使用量で学習できるようです。
+
+あらかじめaccelerate configでfp16を指定し、オプションでmixed_precision="fp16"としてください（bf16では動作しません）。
+
+メモリ使用量を最小化するためには、xformers、use_8bit_adam、gradient_checkpointingの各オプションを指定し、train_batch_sizeを1としてください。
+（余裕があるようならtrain_batch_sizeを段階的に増やすと若干精度が上がるはずです。）
+
+PyTorchのソースにパッチを当てて無理やり実現しています（PyTorch 1.12.1と1.13.0で確認）。精度はかなり落ちますし、途中で学習失敗する確率も高くなります。学習率やステップ数の設定もシビアなようです。それらを認識したうえで自己責任でお使いください。
+-->
+
+# fine tuning特有のその他の主なオプション
+
+すべてのオプションについては別文書を参照してください。
+
+## `train_text_encoder`
+Text Encoderも学習対象とします。メモリ使用量が若干増加します。
+
+通常のfine tuningではText Encoderは学習対象としませんが（恐らくText Encoderの出力に従うようにU-Netを学習するため）、学習データ数が少ない場合には、DreamBoothのようにText Encoder側に学習させるのも有効的なようです。
+
+## `diffusers_xformers`
+スクリプト独自のxformers置換機能ではなくDiffusersのxformers機能を利用します。Hypernetworkの学習はできなくなります。
diff --git a/docs/gen_img_README-ja.md b/docs/gen_img_README-ja.md
new file mode 100644
index 0000000000000000000000000000000000000000..cf35f1df777b65c6b1751d94aaac1a4870987716
--- /dev/null
+++ b/docs/gen_img_README-ja.md
@@ -0,0 +1,454 @@
+SD 1.xおよび2.xのモデル、当リポジトリで学習したLoRA、ControlNet（v1.0のみ動作確認）などに対応した、Diffusersベースの推論（画像生成）スクリプトです。コマンドラインから用います。
+
+# 概要
+
+* Diffusers (v0.10.2) ベースの推論（画像生成）スクリプト。
+* SD 1.xおよび2.x (base/v-parameterization)モデルに対応。
+* txt2img、img2img、inpaintingに対応。
+* 対話モード、およびファイルからのプロンプト読み込み、連続生成に対応。
+* プロンプト1行あたりの生成枚数を指定可能。
+* 全体の繰り返し回数を指定可能。
+* `fp16`だけでなく`bf16`にも対応。
+* xformersに対応し高速生成が可能。
+    * xformersにより省メモリ生成を行いますが、Automatic 1111氏のWeb UIほど最適化していないため、512*512の画像生成でおおむね6GB程度のVRAMを使用します。
+* プロンプトの225トークンへの拡張。ネガティブプロンプト、重みづけに対応。
+* Diffusersの各種samplerに対応（Web UIよりもsampler数は少ないです）。
+* Text Encoderのclip skip（最後からn番目の層の出力を用いる）に対応。
+* VAEの別途読み込み。
+* CLIP Guided Stable Diffusion、VGG16 Guided Stable Diffusion、Highres. fix、upscale対応。
+    * Highres. fixはWeb UIの実装を全く確認していない独自実装のため、出力結果は異なるかもしれません。
+* LoRA対応。適用率指定、複数LoRA同時利用、重みのマージに対応。
+    * Text EncoderとU-Netで別の適用率を指定することはできません。
+* Attention Coupleに対応。
+* ControlNet v1.0に対応。
+* 途中でモデルを切り替えることはできませんが、バッチファイルを組むことで対応できます。
+* 個人的に欲しくなった機能をいろいろ追加。
+
+機能追加時にすべてのテストを行っているわけではないため、以前の機能に影響が出て一部機能が動かない可能性があります。何か問題があればお知らせください。
+
+# 基本的な使い方
+
+## 対話モードでの画像生成
+
+以下のように入力してください。
+
+```batchfile
+python gen_img_diffusers.py --ckpt <モデル名> --outdir <画像出力先> --xformers --fp16 --interactive
+```
+
+`--ckpt`オプションにモデル（Stable Diffusionのcheckpointファイル、またはDiffusersのモデルフォルダ）、`--outdir`オプションに画像の出力先フォルダを指定します。
+
+`--xformers`オプションでxformersの使用を指定します（xformersを使わない場合は外してください）。`--fp16`オプションでfp16（単精度）での推論を行います。RTX 30系のGPUでは `--bf16`オプションでbf16（bfloat16）での推論を行うこともできます。
+
+`--interactive`オプションで対話モードを指定しています。
+
+Stable Diffusion 2.0（またはそこからの追加学習モデル）を使う場合は`--v2`オプションを追加してください。v-parameterizationを使うモデル（`768-v-ema.ckpt`およびそこからの追加学習モデル）を使う場合はさらに`--v_parameterization`を追加してください。
+
+`--v2`の指定有無が間違っているとモデル読み込み時にエラーになります。`--v_parameterization`の指定有無が間違っていると茶色い画像が表示されます。
+
+`Type prompt:`と表示されたらプロンプトを入力してください。
+
+![image](https://user-images.githubusercontent.com/52813779/235343115-f3b8ac82-456d-4aab-9724-0cc73c4534aa.png)
+
+※画像が表示されずエラーになる場合、headless（画面表示機能なし）のOpenCVがインストールされているかもしれません。`pip install opencv-python`として通常のOpenCVを入れてください。または`--no_preview`オプションで画像表示を止めてください。
+
+画像ウィンドウを選択してから何らかのキーを押すとウィンドウが閉じ、次のプロンプトが入力できます。プロンプトでCtrl+Z、エンターの順に打鍵するとスクリプトを閉じます。
+
+## 単一のプロンプトで画像を一括生成
+
+以下のように入力します（実際には1行で入力します）。
+
+```batchfile
+python gen_img_diffusers.py --ckpt <モデル名> --outdir <画像出力先> 
+    --xformers --fp16 --images_per_prompt <生成枚数> --prompt "<プロンプト>"
+```
+
+`--images_per_prompt`オプションで、プロンプト1件当たりの生成枚数を指定します。`--prompt`オプションでプロンプトを指定します。スペースを含む場合はダブルクォーテーションで囲んでください。
+
+`--batch_size`オプションでバッチサイズを指定できます（後述）。
+
+## ファイルからプロンプトを読み込み一括生成
+
+以下のように入力します。
+
+```batchfile
+python gen_img_diffusers.py --ckpt <モデル名> --outdir <画像出力先> 
+    --xformers --fp16 --from_file <プロンプトファイル名>
+```
+
+`--from_file`オプションで、プロンプトが記述されたファイルを指定します。1行1プロンプトで記述してください。`--images_per_prompt`オプションを指定して1行あたり生成枚数を指定できます。
+
+## ネガティブプロンプト、重みづけの使用
+
+プロンプトオプション（プロンプト内で`--x`のように指定、後述）で`--n`を書くと、以降がネガティブプロンプトとなります。
+
+またAUTOMATIC1111氏のWeb UIと同様の `()` や` []` 、`(xxx:1.3)` などによる重みづけが可能です（実装はDiffusersの[Long Prompt Weighting Stable Diffusion](https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#long-prompt-weighting-stable-diffusion)からコピーしたものです）。
+
+コマンドラインからのプロンプト指定、ファイルからのプロンプト読み込みでも同様に指定できます。
+
+![image](https://user-images.githubusercontent.com/52813779/235343128-e79cd768-ec59-46f5-8395-fce9bdc46208.png)
+
+# 主なオプション
+
+コマンドラインから指定してください。
+
+## モデルの指定
+
+- `--ckpt <モデル名>`：モデル名を指定します。`--ckpt`オプションは必須です。Stable Diffusionのcheckpointファイル、またはDiffusersのモデルフォルダ、Hugging FaceのモデルIDを指定できます。
+
+- `--v2`：Stable Diffusion 2.x系のモデルを使う場合に指定します。1.x系の場合には指定不要です。
+
+- `--v_parameterization`：v-parameterizationを使うモデルを使う場合に指定します（`768-v-ema.ckpt`およびそこからの追加学習モデル、Waifu Diffusion v1.5など）。
+    
+    `--v2`の指定有無が間違っているとモデル読み込み時にエラーになります。`--v_parameterization`の指定有無が間違っていると茶色い画像が表示されます。
+
+- `--vae`：使用するVAEを指定します。未指定時はモデル内のVAEを使用します。
+
+## 画像生成と出力
+
+- `--interactive`：インタラクティブモードで動作します。プロンプトを入力すると画像が生成されます。
+
+- `--prompt <プロンプト>`：プロンプトを指定します。スペースを含む場合はダブルクォーテーションで囲んでください。
+
+- `--from_file <プロンプトファイル名>`：プロンプトが記述されたファイルを指定します。1行1プロンプトで記述してください。なお画像サイズやguidance scaleはプロンプトオプション（後述）で指定できます。
+
+- `--W <画像幅>`：画像の幅を指定します。デフォルトは`512`です。
+
+- `--H <画像高さ>`：画像の高さを指定します。デフォルトは`512`です。
+
+- `--steps <ステップ数>`：サンプリングステップ数を指定します。デフォルトは`50`です。
+
+- `--scale <ガイダンススケール>`：unconditionalガイダンススケールを指定します。デフォルトは`7.5`です。
+
+- `--sampler <サンプラー名>`：サンプラーを指定します。デフォルトは`ddim`です。Diffusersで提供されているddim、pndm、dpmsolver、dpmsolver+++、lms、euler、euler_a、が指定可能です（後ろの三つはk_lms、k_euler、k_euler_aでも指定できます）。
+
+- `--outdir <画像出力先フォルダ>`：画像の出力先を指定します。
+
+- `--images_per_prompt <生成枚数>`：プロンプト1件当たりの生成枚数を指定します。デフォルトは`1`です。
+
+- `--clip_skip <スキップ数>`：CLIPの後ろから何番目の層を使うかを指定します。省略時は最後の層を使います。
+
+- `--max_embeddings_multiples <倍数>`：CLIPの入出力長をデフォルト（75）の何倍にするかを指定します。未指定時は75のままです。たとえば3を指定すると入出力長が225になります。
+
+- `--negative_scale` : uncoditioningのguidance scaleを個別に指定します。[gcem156氏のこちらの記事](https://note.com/gcem156/n/ne9a53e4a6f43)を参考に実装したものです。
+
+## メモリ使用量や生成速度の調整
+
+- `--batch_size <バッチサイズ>`：バッチサイズを指定します。デフォルトは`1`です。バッチサイズが大きいとメモリを多く消費しますが、生成速度が速くなります。
+
+- `--vae_batch_size <VAEのバッチサイズ>`：VAEのバッチサイズを指定します。デフォルトはバッチサイズと同じです。
+    VAEのほうがメモリを多く消費するため、デノイジング後（stepが100%になった後）でメモリ不足になる場合があります。このような場合にはVAEのバッチサイズを小さくしてください。
+
+- `--xformers`：xformersを使う場合に指定します。
+
+- `--fp16`：fp16（単精度）での推論を行います。`fp16`と`bf16`をどちらも指定しない場合はfp32（単精度）での推論を行います。
+
+- `--bf16`：bf16（bfloat16）での推論を行います。RTX 30系のGPUでのみ指定可能です。`--bf16`オプションはRTX 30系以外のGPUではエラーになります。`fp16`よりも`bf16`のほうが推論結果がNaNになる（真っ黒の画像になる）可能性が低いようです。
+
+## 追加ネットワーク（LoRA等）の使用
+
+- `--network_module`：使用する追加ネットワークを指定します。LoRAの場合は`--network_module networks.lora`と指定します。複数のLoRAを使用する場合は`--network_module networks.lora networks.lora networks.lora`のように指定します。
+
+- `--network_weights`：使用する追加ネットワークの重みファイルを指定します。`--network_weights model.safetensors`のように指定します。複数のLoRAを使用する場合は`--network_weights model1.safetensors model2.safetensors model3.safetensors`のように指定します。引数の数は`--network_module`で指定した数と同じにしてください。
+
+- `--network_mul`：使用する追加ネットワークの重みを何倍にするかを指定します。デフォルトは`1`です。`--network_mul 0.8`のように指定します。複数のLoRAを使用する場合は`--network_mul 0.4 0.5 0.7`のように指定します。引数の数は`--network_module`で指定した数と同じにしてください。
+
+- `--network_merge`：使用する追加ネットワークの重みを`--network_mul`に指定した重みであらかじめマージします。`--network_pre_calc` と同時に使用できません。プロンプトオプションの`--am`、およびRegional LoRAは使用できなくなりますが、LoRA未使用時と同じ程度まで生成が高速化されます。
+
+- `--network_pre_calc`：使用する追加ネットワークの重みを生成ごとにあらかじめ計算します。プロンプトオプションの`--am`が使用できます。LoRA未使用時と同じ程度まで生成は高速化されますが、生成前に重みを計算する時間が必要で、またメモリ使用量も若干増加します。Regional LoRA使用時は無効になります 。
+
+# 主なオプションの指定例
+
+次は同一プロンプトで64枚をバッチサイズ4で一括生成する例です。
+
+```batchfile
+python gen_img_diffusers.py --ckpt model.ckpt --outdir outputs 
+    --xformers --fp16 --W 512 --H 704 --scale 12.5 --sampler k_euler_a 
+    --steps 32 --batch_size 4 --images_per_prompt 64 
+    --prompt "beautiful flowers --n monochrome"
+```
+
+次はファイルに書かれたプロンプトを、それぞれ10枚ずつ、バッチサイズ4で一括生成する例です。
+
+```batchfile
+python gen_img_diffusers.py --ckpt model.ckpt --outdir outputs 
+    --xformers --fp16 --W 512 --H 704 --scale 12.5 --sampler k_euler_a 
+    --steps 32 --batch_size 4 --images_per_prompt 10 
+    --from_file prompts.txt
+```
+
+Textual Inversion（後述）およびLoRAの使用例です。
+
+```batchfile
+python gen_img_diffusers.py --ckpt model.safetensors 
+    --scale 8 --steps 48 --outdir txt2img --xformers 
+    --W 512 --H 768 --fp16 --sampler k_euler_a 
+    --textual_inversion_embeddings goodembed.safetensors negprompt.pt 
+    --network_module networks.lora networks.lora 
+    --network_weights model1.safetensors model2.safetensors 
+    --network_mul 0.4 0.8 
+    --clip_skip 2 --max_embeddings_multiples 1 
+    --batch_size 8 --images_per_prompt 1 --interactive
+```
+
+# プロンプトオプション
+
+プロンプト内で、`--n`のように「ハイフンふたつ+アルファベットn文字」でプロンプトから各種オプションの指定が可能です。対話モード、コマンドライン、ファイル、いずれからプロンプトを指定する場合でも有効です。
+
+プロンプトのオプション指定`--n`の前後にはスペースを入れてください。
+
+- `--n`：ネガティブプロンプトを指定します。
+
+- `--w`：画像幅を指定します。コマンドラインからの指定を上書きします。
+
+- `--h`：画像高さを指定します。コマンドラインからの指定を上書きします。
+
+- `--s`：ステップ数を指定します。コマンドラインからの指定を上書きします。
+
+- `--d`：この画像の乱数seedを指定します。`--images_per_prompt`を指定している場合は「--d 1,2,3,4」のようにカンマ区切りで複数指定してください。
+    ※様々な理由により、Web UIとは同じ乱数seedでも生成される画像が異なる場合があります。
+
+- `--l`：guidance scaleを指定します。コマンドラインからの指定を上書きします。
+
+- `--t`：img2img（後述）のstrengthを指定します。コマンドラインからの指定を上書きします。
+
+- `--nl`：ネガティブプロンプトのguidance scaleを指定します（後述）。コマンドラインからの指定を上書きします。
+
+- `--am`：追加ネットワークの重みを指定します。コマンドラインからの指定を上書きします。複数の追加ネットワークを使用する場合は`--am 0.8,0.5,0.3`のように __カンマ区切りで__ 指定します。
+
+※これらのオプションを指定すると、バッチサイズよりも小さいサイズでバッチが実行される場合があります（これらの値が異なると一括生成できないため）。（あまり気にしなくて大丈夫ですが、ファイルからプロンプトを読み込み生成する場合は、これらの値が同一のプロンプトを並べておくと効率が良くなります。）
+
+例：
+```
+(masterpiece, best quality), 1girl, in shirt and plated skirt, standing at street under cherry blossoms, upper body, [from below], kind smile, looking at another, [goodembed] --n realistic, real life, (negprompt), (lowres:1.1), (worst quality:1.2), (low quality:1.1), bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, normal quality, jpeg artifacts, signature, watermark, username, blurry --w 960 --h 640 --s 28 --d 1
+```
+
+![image](https://user-images.githubusercontent.com/52813779/235343446-25654172-fff4-4aaf-977a-20d262b51676.png)
+
+# img2img
+
+## オプション
+
+- `--image_path`：img2imgに利用する画像を指定します。`--image_path template.png`のように指定します。フォルダを指定すると、そのフォルダの画像を順次利用します。
+
+- `--strength`：img2imgのstrengthを指定します。`--strength 0.8`のように指定します。デフォルトは`0.8`です。
+
+- `--sequential_file_name`：ファイル名を連番にするかどうかを指定します。指定すると生成されるファイル名が`im_000001.png`からの連番になります。
+
+- `--use_original_file_name`：指定すると生成ファイル名がオリジナルのファイル名と同じになります。
+
+## コマンドラインからの実行例
+
+```batchfile
+python gen_img_diffusers.py --ckpt trinart_characters_it4_v1_vae_merged.ckpt 
+    --outdir outputs --xformers --fp16 --scale 12.5 --sampler k_euler --steps 32 
+    --image_path template.png --strength 0.8 
+    --prompt "1girl, cowboy shot, brown hair, pony tail, brown eyes, 
+          sailor school uniform, outdoors 
+          --n lowres, bad anatomy, bad hands, error, missing fingers, cropped, 
+          worst quality, low quality, normal quality, jpeg artifacts, (blurry), 
+          hair ornament, glasses" 
+    --batch_size 8 --images_per_prompt 32
+```
+
+`--image_path`オプションにフォルダを指定すると、そのフォルダの画像を順次読み込みます。生成される枚数は画像枚数ではなく、プロンプト数になりますので、`--images_per_promptPPオプションを指定してimg2imgする画像の枚数とプロンプト数を合わせてください。
+
+ファイルはファイル名でソートして読み込みます。なおソート順は文字列順となりますので（`1.jpg→2.jpg→10.jpg`ではなく`1.jpg→10.jpg→2.jpg`の順）、頭を0埋めするなどしてご対応ください（`01.jpg→02.jpg→10.jpg`）。
+
+## img2imgを利用したupscale
+
+img2img時にコマンドラインオプションの`--W`と`--H`で生成画像サイズを指定すると、元画像をそのサイズにリサイズしてからimg2imgを行います。
+
+またimg2imgの元画像がこのスクリプトで生成した画像の場合、プロンプトを省略すると、元画像のメタデータからプロンプトを取得しそのまま用います。これによりHighres. fixの2nd stageの動作だけを行うことができます。
+
+## img2img時のinpainting
+
+画像およびマスク画像を指定してinpaintingできます（inpaintingモデルには対応しておらず、単にマスク領域を対象にimg2imgするだけです）。
+
+オプションは以下の通りです。
+
+- `--mask_image`：マスク画像を指定します。`--img_path`と同様にフォルダを指定すると、そのフォルダの画像を順次利用します。
+
+マスク画像はグレースケール画像で、白の部分がinpaintingされます。境界をグラデーションしておくとなんとなく滑らかになりますのでお勧めです。
+
+![image](https://user-images.githubusercontent.com/52813779/235343795-9eaa6d98-02ff-4f32-b089-80d1fc482453.png)
+
+# その他の機能
+
+## Textual Inversion
+
+`--textual_inversion_embeddings`オプションで使用するembeddingsを指定します（複数指定可）。拡張子を除いたファイル名をプロンプト内で使用することで、そのembeddingsを利用します（Web UIと同様の使用法です）。ネガティブプロンプト内でも使用できます。
+
+モデルとして、当リポジトリで学習したTextual Inversionモデル、およびWeb UIで学習したTextual Inversionモデル（画像埋め込みは非対応）を利用できます
+
+## Extended Textual Inversion
+
+`--textual_inversion_embeddings`の代わりに`--XTI_embeddings`オプションを指定してください。使用法は`--textual_inversion_embeddings`と同じです。
+
+## Highres. fix
+
+AUTOMATIC1111氏のWeb UIにある機能の類似機能です（独自実装のためもしかしたらいろいろ異なるかもしれません）。最初に小さめの画像を生成し、その画像を元にimg2imgすることで、画像全体の破綻を防ぎつつ大きな解像度の画像を生成します。
+
+2nd stageのstep数は`--steps` と`--strength`オプションの値から計算されます（`steps*strength`）。
+
+img2imgと併用できません。
+
+以下のオプションがあります。
+
+- `--highres_fix_scale`：Highres. fixを有効にして、1st stageで生成する画像のサイズを、倍率で指定します。最終出力が1024x1024で、最初に512x512の画像を生成する場合は`--highres_fix_scale 0.5`のように指定します。Web UI出の指定の逆数になっていますのでご注意ください。
+
+- `--highres_fix_steps`：1st stageの画像のステップ数を指定します。デフォルトは`28`です。
+
+- `--highres_fix_save_1st`：1st stageの画像を保存するかどうかを指定します。
+
+- `--highres_fix_latents_upscaling`：指定すると2nd stageの画像生成時に1st stageの画像をlatentベースでupscalingします（bilinearのみ対応）。未指定時は画像をLANCZOS4でupscalingします。
+
+- `--highres_fix_upscaler`：2nd stageに任意のupscalerを利用します。現在は`--highres_fix_upscaler tools.latent_upscaler` のみ対応しています。
+
+- `--highres_fix_upscaler_args`：`--highres_fix_upscaler`で指定したupscalerに渡す引数を指定します。
+    `tools.latent_upscaler`の場合は、`--highres_fix_upscaler_args "weights=D:\Work\SD\Models\others\etc\upscaler-v1-e100-220.safetensors"`のように重みファイルを指定します。 
+
+コマンドラインの例です。
+
+```batchfile
+python gen_img_diffusers.py  --ckpt trinart_characters_it4_v1_vae_merged.ckpt
+    --n_iter 1 --scale 7.5 --W 1024 --H 1024 --batch_size 1 --outdir ../txt2img 
+    --steps 48 --sampler ddim --fp16 
+    --xformers 
+    --images_per_prompt 1  --interactive 
+    --highres_fix_scale 0.5 --highres_fix_steps 28 --strength 0.5
+```
+
+## ControlNet
+
+現在はControlNet 1.0のみ動作確認しています。プリプロセスはCannyのみサポートしています。
+
+以下のオプションがあります。
+
+- `--control_net_models`：ControlNetのモデルファイルを指定します。
+    複数指定すると、それらをstepごとに切り替えて利用します（Web UIのControlNet拡張の実装と異なります）。diffと通常の両方をサポートします。
+
+- `--guide_image_path`：ControlNetに使うヒント画像を指定します。`--img_path`と同様にフォルダを指定すると、そのフォルダの画像を順次利用します。Canny以外のモデルの場合には、あらかじめプリプロセスを行っておいてください。
+
+- `--control_net_preps`：ControlNetのプリプロセスを指定します。`--control_net_models`と同様に複数指定可能です。現在はcannyのみ対応しています。対象モデルでプリプロセスを使用しない場合は `none` を指定します。
+   cannyの場合 `--control_net_preps canny_63_191`のように、閾値1と2を'_'で区切って指定できます。
+
+- `--control_net_weights`：ControlNetの適用時の重みを指定します（`1.0`で通常、`0.5`なら半分の影響力で適用）。`--control_net_models`と同様に複数指定可能です。
+
+- `--control_net_ratios`：ControlNetを適用するstepの範囲を指定します。`0.5`の場合は、step数の半分までControlNetを適用します。`--control_net_models`と同様に複数指定可能です。
+
+コマンドラインの例です。
+
+```batchfile
+python gen_img_diffusers.py --ckpt model_ckpt --scale 8 --steps 48 --outdir txt2img --xformers 
+    --W 512 --H 768 --bf16 --sampler k_euler_a 
+    --control_net_models diff_control_sd15_canny.safetensors --control_net_weights 1.0 
+    --guide_image_path guide.png --control_net_ratios 1.0 --interactive
+```
+
+## Attention Couple + Reginal LoRA
+
+プロンプトをいくつかの部分に分割し、それぞれのプロンプトを画像内のどの領域に適用するかを指定できる機能です。個別のオプションはありませんが、`mask_path`とプロンプトで指定します。
+
+まず、プロンプトで` AND `を利用して、複数部分を定義します。最初の3つに対して領域指定ができ、以降の部分は画像全体へ適用されます。ネガティブプロンプトは画像全体に適用されます。
+
+以下ではANDで3つの部分を定義しています。
+
+```
+shs 2girls, looking at viewer, smile AND bsb 2girls, looking back AND 2girls --n bad quality, worst quality
+```
+
+次にマスク画像を用意します。マスク画像はカラーの画像で、RGBの各チャネルがプロンプトのANDで区切られた部分に対応します。またあるチャネルの値がすべて0の場合、画像全体に適用されます。
+
+上記の例では、Rチャネルが`shs 2girls, looking at viewer, smile`、Gチャネルが`bsb 2girls, looking back`に、Bチャネルが`2girls`に対応します。次のようなマスク画像を使用すると、Bチャネルに指定がありませんので、`2girls`は画像全体に適用されます。
+
+![image](https://user-images.githubusercontent.com/52813779/235343061-b4dc9392-3dae-4831-8347-1e9ae5054251.png)
+
+マスク画像は`--mask_path`で指定します。現在は1枚のみ対応しています。指定した画像サイズに自動的にリサイズされ適用されます。
+
+ControlNetと組み合わせることも可能です（細かい位置指定にはControlNetとの組み合わせを推奨します）。
+
+LoRAを指定すると、`--network_weights`で指定した複数のLoRAがそれぞれANDの各部分に対応します。現在の制約として、LoRAの数はANDの部分の数と同じである必要があります。
+
+## CLIP Guided Stable Diffusion
+
+DiffusersのCommunity Examplesの[こちらのcustom pipeline](https://github.com/huggingface/diffusers/blob/main/examples/community/README.md#clip-guided-stable-diffusion)からソースをコピー、変更したものです。
+
+通常のプロンプトによる生成指定に加えて、追加でより大規模のCLIPでプロンプトのテキストの特徴量を取得し、生成中の画像の特徴量がそのテキストの特徴量に近づくよう、生成される画像をコントロールします（私のざっくりとした理解です）。大きめのCLIPを使いますのでVRAM使用量はかなり増加し（VRAM 8GBでは512*512でも厳しいかもしれません）、生成時間も掛かります。
+
+なお選択できるサンプラーはDDIM、PNDM、LMSのみとなります。
+
+`--clip_guidance_scale`オプションにどの程度、CLIPの特徴量を反映するかを数値で指定します。先のサンプルでは100になっていますので、そのあたりから始めて増減すると良いようです。
+
+デフォルトではプロンプトの先頭75トークン（重みづけの特殊文字を除く）がCLIPに渡されます。プロンプトの`--c`オプションで、通常のプロンプトではなく、CLIPに渡すテキストを別に指定できます（たとえばCLIPはDreamBoothのidentifier（識別子）や「1girl」などのモデル特有の単語は認識できないと思われますので、それらを省いたテキストが良いと思われます）。
+
+コマンドラインの例です。
+
+```batchfile
+python gen_img_diffusers.py  --ckpt v1-5-pruned-emaonly.ckpt --n_iter 1 
+    --scale 2.5 --W 512 --H 512 --batch_size 1 --outdir ../txt2img --steps 36  
+    --sampler ddim --fp16 --opt_channels_last --xformers --images_per_prompt 1  
+    --interactive --clip_guidance_scale 100
+```
+
+## CLIP Image Guided Stable Diffusion
+
+テキストではなくCLIPに別の画像を渡し、その特徴量に近づくよう生成をコントロールする機能です。`--clip_image_guidance_scale`オプションで適用量の数値を、`--guide_image_path`オプションでguideに使用する画像（ファイルまたはフォルダ）を指定してください。
+
+コマンドラインの例です。
+
+```batchfile
+python gen_img_diffusers.py  --ckpt trinart_characters_it4_v1_vae_merged.ckpt
+    --n_iter 1 --scale 7.5 --W 512 --H 512 --batch_size 1 --outdir ../txt2img 
+    --steps 80 --sampler ddim --fp16 --opt_channels_last --xformers 
+    --images_per_prompt 1  --interactive  --clip_image_guidance_scale 100 
+    --guide_image_path YUKA160113420I9A4104_TP_V.jpg
+```
+
+### VGG16 Guided Stable Diffusion
+
+指定した画像に近づくように画像生成する機能です。通常のプロンプトによる生成指定に加えて、追加でVGG16の特徴量を取得し、生成中の画像が指定したガイド画像に近づくよう、生成される画像をコントロールします。img2imgでの使用をお勧めします（通常の生成では画像がぼやけた感じになります）。CLIP Guided Stable Diffusionの仕組みを流用した独自の機能です。またアイデアはVGGを利用したスタイル変換から拝借しています。
+
+なお選択できるサンプラーはDDIM、PNDM、LMSのみとなります。
+
+`--vgg16_guidance_scale`オプションにどの程度、VGG16特徴量を反映するかを数値で指定します。試した感じでは100くらいから始めて増減すると良いようです。`--guide_image_path`オプションでguideに使用する画像（ファイルまたはフォルダ）を指定してください。
+
+複数枚の画像を一括でimg2img変換し、元画像をガイド画像とする場合、`--guide_image_path`と`--image_path`に同じ値を指定すればOKです。
+
+コマンドラインの例です。
+
+```batchfile
+python gen_img_diffusers.py --ckpt wd-v1-3-full-pruned-half.ckpt 
+    --n_iter 1 --scale 5.5 --steps 60 --outdir ../txt2img 
+    --xformers --sampler ddim --fp16 --W 512 --H 704 
+    --batch_size 1 --images_per_prompt 1 
+    --prompt "picturesque, 1girl, solo, anime face, skirt, beautiful face 
+        --n lowres, bad anatomy, bad hands, error, missing fingers, 
+        cropped, worst quality, low quality, normal quality, 
+        jpeg artifacts, blurry, 3d, bad face, monochrome --d 1" 
+    --strength 0.8 --image_path ..\src_image
+    --vgg16_guidance_scale 100 --guide_image_path ..\src_image 
+```
+
+`--vgg16_guidance_layerPで特徴量取得に使用するVGG16のレイヤー番号を指定できます（デフォルトは20でconv4-2のReLUです）。上の層ほど画風を表現し、下の層ほどコンテンツを表現するといわれています。
+
+![image](https://user-images.githubusercontent.com/52813779/235343813-3c1f0d7a-4fb3-4274-98e4-b92d76b551df.png)
+
+# その他のオプション
+
+- `--no_preview` : 対話モードでプレビュー画像を表示しません。OpenCVがインストールされていない場合や、出力されたファイルを直接確認する場合に指定してください。
+
+- `--n_iter` : 生成を繰り返す回数を指定します。デフォルトは1です。プロンプトをファイルから読み込むとき、複数回の生成を行いたい場合に指定します。
+
+- `--tokenizer_cache_dir` : トークナイザーのキャッシュディレクトリを指定します。（作業中）
+
+- `--seed` : 乱数seedを指定します。1枚生成時はその画像のseed、複数枚生成時は各画像のseedを生成するための乱数のseedになります（`--from_file`で複数画像生成するとき、`--seed`オプションを指定すると複数回実行したときに各画像が同じseedになります）。
+
+- `--iter_same_seed` : プロンプトに乱数seedの指定がないとき、`--n_iter`の繰り返し内ではすべて同じseedを使います。`--from_file`で指定した複数のプロンプト間でseedを統一して比較するときに使います。
+
+- `--diffusers_xformers` : Diffuserのxformersを使用します。
+
+- `--opt_channels_last` : 推論時にテンソルのチャンネルを最後に配置します。場合によっては高速化されることがあります。
+
+- `--network_show_meta` : 追加ネットワークのメタデータを表示します。
+
diff --git a/docs/image_folder_structure.md b/docs/image_folder_structure.md
new file mode 100644
index 0000000000000000000000000000000000000000..effdc7cd67906c9efc6302952a50c25f6f83658d
--- /dev/null
+++ b/docs/image_folder_structure.md
@@ -0,0 +1,59 @@
+# Drambootd, Lora and TI image folder structure
+
+To ensure successful training with Kohya, it is crucial to follow a specific folder structure that provides the necessary image repeats. Please adhere to the following structure precisely:
+
+Folder Structure Example:
+
+```txt
+c:
+|
+├──images
+|   |
+|   ├── 30_cat
+|   |   |
+|   |   ├── image1.jpg
+|   |   ├── image1.txt
+|   |   ├── image2.png
+|   |   └── image2.txt
+|   |
+|   ├── 30_dog
+|   |   |
+|   |   ├── image1.jpg
+|   |   ├── image1.txt
+|   |   ├── image2.png
+|   |   └── image2.txt
+|   |
+|   └── 40_black mamba
+|       |
+|       ├── image1.jpg
+|       ├── image1.txt
+|       ├── image2.png
+|       └── image2.txt
+|
+├──regularization
+|   |
+|   ├── 1_cat
+|   |   |
+|   |   ├── reg1.jpg
+|   |   ├── reg2.jpg
+|   |
+|   ├── 1_dog
+|   |   |
+|   |   ├── reg1.jpg
+|   |   ├── reg2.jpg
+|   |
+|   └── 1_black mamba
+|       |
+|       ├── reg1.jpg
+|       ├── reg2.jpg
+
+```
+
+Please note the following important information regarding file extensions and their impact on concept names during model training:
+
+If a file with a .txt or .caption extension and the same name as an image is present in the image subfolder, it will take precedence over the concept name during the model training process.
+For example, if there is an image file named image1.jpg in the 30_cat subfolder, and there is a corresponding text file named image1.txt or image1.caption in the same subfolder, the concept name used during training will be determined by the content of that text file rather than the subfolder name.
+
+Ensure that the content of such text files accurately reflects the desired concept name or any relevant caption information associated with the corresponding image.
+
+By considering this information and maintaining the proper folder structure, including any necessary text or caption files, you can ensure a smooth and effective training process with Kohya.
\ No newline at end of file
diff --git a/docs/train_README-ja.md b/docs/train_README-ja.md
new file mode 100644
index 0000000000000000000000000000000000000000..158363b397a8d925e41c49f96ff5cbd6699d1845
--- /dev/null
+++ b/docs/train_README-ja.md
@@ -0,0 +1,1002 @@
+__ドキュメント更新中のため記述に誤りがあるかもしれません。__
+
+# 学習について、共通編
+
+当リポジトリではモデルのfine tuning、DreamBooth、およびLoRAとTextual Inversion（[XTI:P+](https://github.com/kohya-ss/sd-scripts/pull/327)を含む）の学習をサポートします。この文書ではそれらに共通する、学習データの準備方法やオプション等について説明します。
+
+# 概要
+
+あらかじめこのリポジトリのREADMEを参照し、環境整備を行ってください。
+
+
+以下について説明します。
+
+1. 学習データの準備について（設定ファイルを用いる新形式）
+1. 学習で使われる用語のごく簡単な解説
+1. 以前の指定形式（設定ファイルを用いずコマンドラインから指定）
+1. 学習途中のサンプル画像生成
+1. 各スクリプトで共通の、よく使われるオプション
+1. fine tuning 方式のメタデータ準備：キャプションニングなど
+
+1.だけ実行すればとりあえず学習は可能です（学習については各スクリプトのドキュメントを参照）。2.以降は必要に応じて参照してください。
+
+
+# 学習データの準備について
+
+任意のフォルダ（複数でも可）に学習データの画像ファイルを用意しておきます。`.png`, `.jpg`, `.jpeg`, `.webp`, `.bmp` をサポートします。リサイズなどの前処理は基本的に必要ありません。
+
+ただし学習解像度（後述）よりも極端に小さい画像は使わないか、あらかじめ超解像AIなどで拡大しておくことをお勧めします。また極端に大きな画像（3000x3000ピクセル程度？）よりも大きな画像はエラーになる場合があるようですので事前に縮小してください。
+
+学習時には、モデルに学ばせる画像データを整理し、スクリプトに対して指定する必要があります。学習データの数、学習対象、キャプション（画像の説明）が用意できるか否かなどにより、いくつかの方法で学習データを指定できます。以下の方式があります（それぞれの名前は一般的なものではなく、当リポジトリ独自の定義です）。正則化画像については後述します。
+
+1. DreamBooth、class+identifier方式（正則化画像使用可）
+
+    特定の単語 (identifier) に学習対象を紐づけるように学習します。キャプションを用意する必要はありません。たとえば特定のキャラを学ばせる場合に使うとキャプションを用意する必要がない分、手軽ですが、髪型や服装、背景など学習データの全要素が identifier に紐づけられて学習されるため、生成時のプロンプトで服が変えられない、といった事態も起こりえます。
+
+1. DreamBooth、キャプション方式（正則化画像使用可）
+
+    画像ごとにキャプションが記録されたテキストファイルを用意して学習します。たとえば特定のキャラを学ばせると、画像の詳細をキャプションに記述することで（白い服を着たキャラA、赤い服を着たキャラA、など）キャラとそれ以外の要素が分離され、より厳密にモデルがキャラだけを学ぶことが期待できます。
+
+1. fine tuning方式（正則化画像使用不可）
+
+    あらかじめキャプションをメタデータファイルにまとめます。タグとキャプションを分けて管理したり、学習を高速化するためlatentsを事前キャッシュしたりなどの機能をサポートします（いずれも別文書で説明しています）。（fine tuning方式という名前ですが fine tuning 以外でも使えます。）
+
+学習したいものと使用できる指定方法の組み合わせは以下の通りです。
+
+| 学習対象または方法 | スクリプト | DB / class+identifier | DB / キャプション | fine tuning |
+| ----- | ----- | ----- | ----- | ----- |
+| モデルをfine tuning | `fine_tune.py`| x | x | o |
+| モデルをDreamBooth | `train_db.py`| o | o | x |
+| LoRA | `train_network.py`| o | o | o |
+| Textual Invesion | `train_textual_inversion.py`| o | o | o |
+
+## どれを選ぶか
+
+LoRA、Textual Inversionについては、手軽にキャプションファイルを用意せずに学習したい場合はDreamBooth class+identifier、用意できるならDreamBooth キャプション方式がよいでしょう。学習データの枚数が多く、かつ正則化画像を使用しない場合はfine tuning方式も検討してください。
+
+DreamBoothについても同様ですが、fine tuning方式は使えません。fine tuningの場合はfine tuning方式のみです。
+
+# 各方式の指定方法について
+
+ここではそれぞれの指定方法で典型的なパターンについてだけ説明します。より詳細な指定方法については [データセット設定](./config_README-ja.md) をご覧ください。
+
+# DreamBooth、class+identifier方式（正則化画像使用可）
+
+この方式では、各画像は `class identifier` というキャプションで学習されたのと同じことになります（`shs dog` など）。
+
+## step 1. identifierとclassを決める
+
+学ばせたい対象を結びつける単語identifierと、対象の属するclassを決めます。
+
+（instanceなどいろいろな呼び方がありますが、とりあえず元の論文に合わせます。）
+
+以下ごく簡単に説明します（詳しくは調べてください）。
+
+classは学習対象の一般的な種別です。たとえば特定の犬種を学ばせる場合には、classはdogになります。アニメキャラならモデルによりboyやgirl、1boyや1girlになるでしょう。
+
+identifierは学習対象を識別して学習するためのものです。任意の単語で構いませんが、元論文によると「tokinizerで1トークンになる3文字以下でレアな単語」が良いとのことです。
+
+identifierとclassを使い、たとえば「shs dog」などでモデルを学習することで、学習させたい対象をclassから識別して学習できます。
+
+画像生成時には「shs dog」とすれば学ばせた犬種の画像が生成されます。
+
+（identifierとして私が最近使っているものを参考までに挙げると、``shs sts scs cpc coc cic msm usu ici lvl cic dii muk ori hru rik koo yos wny`` などです。本当は Danbooru Tag に含まれないやつがより望ましいです。）
+
+## step 2. 正則化画像を使うか否かを決め、使う場合には正則化画像を生成する
+
+正則化画像とは、前述のclass全体が、学習対象に引っ張られることを防ぐための画像です（language drift）。正則化画像を使わないと、たとえば `shs 1girl` で特定のキャラクタを学ばせると、単なる `1girl` というプロンプトで生成してもそのキャラに似てきます。これは `1girl` が学習時のキャプションに含まれているためです。
+
+学習対象の画像と正則化画像を同時に学ばせることで、class は class のままで留まり、identifier をプロンプトにつけた時だけ学習対象が生成されるようになります。
+
+LoRAやDreamBoothで特定のキャラだけ出てくればよい場合は、正則化画像を用いなくても良いといえます。
+
+Textual Inversionでは用いなくてよいでしょう（学ばせる token string がキャプションに含まれない場合はなにも学習されないため）。
+
+正則化画像としては、学習対象のモデルで、class 名だけで生成した画像を用いるのが一般的です（たとえば `1girl`）。ただし生成画像の品質が悪い場合には、プロンプトを工夫したり、ネットから別途ダウンロードした画像を用いることもできます。
+
+（正則化画像も学習されるため、その品質はモデルに影響します。）
+
+一般的には数百枚程度、用意するのが望ましいようです（枚数が少ないと class 画像が一般化されずそれらの特徴を学んでしまいます）。
+
+生成画像を使う場合、通常、生成画像のサイズは学習解像度（より正確にはbucketの解像度、後述）にあわせてください。
+
+## step 2. 設定ファイルの記述
+
+テキストファイルを作成し、拡張子を `.toml` にします。たとえば以下のように記述します。
+
+（`#` で始まっている部分はコメントですので、このままコピペしてそのままでもよいですし、削除しても問題ありません。）
+
+```toml
+[general]
+enable_bucket = true                        # Aspect Ratio Bucketingを使うか否か
+
+[[datasets]]
+resolution = 512                            # 学習解像度
+batch_size = 4                              # バッチサイズ
+
+  [[datasets.subsets]]
+  image_dir = 'C:\hoge'                     # 学習用画像を入れたフォルダを指定
+  class_tokens = 'hoge girl'                # identifier class を指定
+  num_repeats = 10                          # 学習用画像の繰り返し回数
+
+  # 以下は正則化画像を用いる場合のみ記述する。用いない場合は削除する
+  [[datasets.subsets]]
+  is_reg = true
+  image_dir = 'C:\reg'                      # 正則化画像を入れたフォルダを指定
+  class_tokens = 'girl'                     # class を指定
+  num_repeats = 1                           # 正則化画像の繰り返し回数、基本的には1でよい
+```
+
+基本的には以下の場所のみ書き換えれば学習できます。
+
+1. 学習解像度
+
+    数値1つを指定すると正方形（`512`なら512x512）、鍵カッコカンマ区切りで2つ指定すると横×縦（`[512,768]`なら512x768）になります。SD1.x系ではもともとの学習解像度は512です。`[512,768]` 等の大きめの解像度を指定すると縦長、横長画像生成時の破綻を小さくできるかもしれません。SD2.x 768系では `768` です。
+
+1. バッチサイズ
+
+    同時に何件のデータを学習するかを指定します。GPUのVRAMサイズ、学習解像度によって変わってきます。詳しくは後述します。またfine tuning/DreamBooth/LoRA等でも変わってきますので各スクリプトの説明もご覧ください。
+
+1. フォルダ指定
+
+    学習用画像、正則化画像（使用する場合のみ）のフォルダを指定します。画像データが含まれているフォルダそのものを指定します。
+
+1. identifier と class の指定
+
+    前述のサンプルの通りです。
+
+1. 繰り返し回数
+
+    後述します。
+
+### 繰り返し回数について
+
+繰り返し回数は、正則化画像の枚数と学習用画像の枚数を調整するために用いられます。正則化画像の枚数は学習用画像よりも多いため、学習用画像を繰り返して枚数を合わせ、1対1の比率で学習できるようにします。
+
+繰り返し回数は「 __学習用画像の繰り返し回数×学習用画像の枚数≧正則化画像の繰り返し回数×正則化画像の枚数__ 」となるように指定してください。
+
+（1 epoch（データが一周すると1 epoch）のデータ数が「学習用画像の繰り返し回数×学習用画像の枚数」となります。正則化画像の枚数がそれより多いと、余った部分の正則化画像は使用されません。）
+
+## step 3. 学習
+
+それぞれのドキュメントを参考に学習を行ってください。
+
+# DreamBooth、キャプション方式（正則化画像使用可）
+
+この方式では各画像はキャプションで学習されます。
+
+## step 1. キャプションファイルを準備する
+
+学習用画像のフォルダに、画像と同じファイル名で、拡張子 `.caption`（設定で変えられます）のファイルを置いてください。それぞれのファイルは1行のみとしてください。エンコーディングは `UTF-8` です。
+
+## step 2. 正則化画像を使うか否かを決め、使う場合には正則化画像を生成する
+
+class+identifier形式と同様です。なお正則化画像にもキャプションを付けることができますが、通常は不要でしょう。
+
+## step 2. 設定ファイルの記述
+
+テキストファイルを作成し、拡張子を `.toml` にします。たとえば以下のように記述します。
+
+```toml
+[general]
+enable_bucket = true                        # Aspect Ratio Bucketingを使うか否か
+
+[[datasets]]
+resolution = 512                            # 学習解像度
+batch_size = 4                              # バッチサイズ
+
+  [[datasets.subsets]]
+  image_dir = 'C:\hoge'                     # 学習用画像を入れたフォルダを指定
+  caption_extension = '.caption'            # キャプションファイルの拡張子　.txt を使う場合には書き換える
+  num_repeats = 10                          # 学習用画像の繰り返し回数
+
+  # 以下は正則化画像を用いる場合のみ記述する。用いない場合は削除する
+  [[datasets.subsets]]
+  is_reg = true
+  image_dir = 'C:\reg'                      # 正則化画像を入れたフォルダを指定
+  class_tokens = 'girl'                     # class を指定
+  num_repeats = 1                           # 正則化画像の繰り返し回数、基本的には1でよい
+```
+
+基本的には以下を場所のみ書き換えれば学習できます。特に記述がない部分は class+identifier 方式と同じです。
+
+1. 学習解像度
+1. バッチサイズ
+1. フォルダ指定
+1. キャプションファイルの拡張子
+
+    任意の拡張子を指定できます。
+1. 繰り返し回数
+
+## step 3. 学習
+
+それぞれのドキュメントを参考に学習を行ってください。
+
+# fine tuning 方式
+
+## step 1. メタデータを準備する
+
+キャプションやタグをまとめた管理用ファイルをメタデータと呼びます。json形式で拡張子は `.json`
+ です。作成方法は長くなりますのでこの文書の末尾に書きました。
+
+## step 2. 設定ファイルの記述
+
+テキストファイルを作成し、拡張子を `.toml` にします。たとえば以下のように記述します。
+
+```toml
+[general]
+shuffle_caption = true
+keep_tokens = 1
+
+[[datasets]]
+resolution = 512                                    # 学習解像度
+batch_size = 4                                      # バッチサイズ
+
+  [[datasets.subsets]]
+  image_dir = 'C:\piyo'                             # 学習用画像を入れたフォルダを指定
+  metadata_file = 'C:\piyo\piyo_md.json'            # メタデータファイル名
+```
+
+基本的には以下を場所のみ書き換えれば学習できます。特に記述がない部分は DreamBooth, class+identifier 方式と同じです。
+
+1. 学習解像度
+1. バッチサイズ
+1. フォルダ指定
+1. メタデータファイル名
+
+    後述の方法で作成したメタデータファイルを指定します。
+
+
+## step 3. 学習
+
+それぞれのドキュメントを参考に学習を行ってください。
+
+# 学習で使われる用語のごく簡単な解説
+
+細かいことは省略していますし私も完全には理解していないため、詳しくは各自お調べください。
+
+## fine tuning（ファインチューニング）
+
+モデルを学習して微調整することを指します。使われ方によって意味が異なってきますが、狭義のfine tuningはStable Diffusionの場合、モデルを画像とキャプションで学習することです。DreamBoothは狭義のfine tuningのひとつの特殊なやり方と言えます。広義のfine tuningは、LoRAやTextual Inversion、Hypernetworksなどを含み、モデルを学習することすべてを含みます。
+
+## ステップ
+
+ざっくりいうと学習データで1回計算すると1ステップです。「学習データのキャプションを今のモデルに流してみて、出てくる画像を学習データの画像と比較し、学習データに近づくようにモデルをわずかに変更する」のが1ステップです。
+
+## バッチサイズ
+
+バッチサイズは1ステップで何件のデータをまとめて計算するかを指定する値です。まとめて計算するため速度は相対的に向上します。また一般的には精度も高くなるといわれています。
+
+`バッチサイズ×ステップ数` が学習に使われるデータの件数になります。そのため、バッチサイズを増やした分だけステップ数を減らすとよいでしょう。
+
+（ただし、たとえば「バッチサイズ1で1600ステップ」と「バッチサイズ4で400ステップ」は同じ結果にはなりません。同じ学習率の場合、一般的には後者のほうが学習不足になります。学習率を多少大きくするか（たとえば `2e-6` など）、ステップ数をたとえば500ステップにするなどして工夫してください。）
+
+バッチサイズを大きくするとその分だけGPUメモリを消費します。メモリが足りなくなるとエラーになりますし、エラーにならないギリギリでは学習速度が低下します。タスクマネージャーや `nvidia-smi` コマンドで使用メモリ量を確認しながら調整するとよいでしょう。
+
+なお、バッチは「一塊のデータ」位の意味です。
+
+## 学習率
+
+ざっくりいうと1ステップごとにどのくらい変化させるかを表します。大きな値を指定するとそれだけ速く学習が進みますが、変化しすぎてモデルが壊れたり、最適な状態にまで至れない場合があります。小さい値を指定すると学習速度は遅くなり、また最適な状態にやはり至れない場合があります。
+
+fine tuning、DreamBoooth、LoRAそれぞれで大きく異なり、また学習データや学習させたいモデル、バッチサイズやステップ数によっても変わってきます。一般的な値から初めて学習状態を見ながら増減してください。
+
+デフォルトでは学習全体を通して学習率は固定です。スケジューラの指定で学習率をどう変化させるか決められますので、それらによっても結果は変わってきます。
+
+## エポック（epoch）
+
+学習データが一通り学習されると（データが一周すると）1 epochです。繰り返し回数を指定した場合は、その繰り返し後のデータが一周すると1 epochです。
+
+1 epochのステップ数は、基本的には `データ件数÷バッチサイズ` ですが、Aspect Ratio Bucketing を使うと微妙に増えます（異なるbucketのデータは同じバッチにできないため、ステップ数が増えます）。
+
+## Aspect Ratio Bucketing
+
+Stable Diffusion のv1は512\*512で学習されていますが、それに加えて256\*1024や384\*640といった解像度でも学習します。これによりトリミングされる部分が減り、より正しくキャプションと画像の関係が学習されることが期待されます。
+
+また任意の解像度で学習するため、事前に画像データの縦横比を統一しておく必要がなくなります。
+
+設定で有効、向こうが切り替えられますが、ここまでの設定ファイルの記述例では有効になっています（`true` が設定されています）。
+
+学習解像度はパラメータとして与えられた解像度の面積（＝メモリ使用量）を超えない範囲で、64ピクセル単位（デフォルト、変更可）で縦横に調整、作成されます。
+
+機械学習では入力サイズをすべて統一するのが一般的ですが、特に制約があるわけではなく、実際は同一のバッチ内で統一されていれば大丈夫です。NovelAIの言うbucketingは、あらかじめ教師データを、アスペクト比に応じた学習解像度ごとに分類しておくことを指しているようです。そしてバッチを各bucket内の画像で作成することで、バッチの画像サイズを統一します。
+
+# 以前の指定形式（設定ファイルを用いずコマンドラインから指定）
+
+`.toml` ファイルを指定せずコマンドラインオプションで指定する方法です。DreamBooth class+identifier方式、DreamBooth キャプション方式、fine tuning方式があります。
+
+## DreamBooth、class+identifier方式
+
+フォルダ名で繰り返し回数を指定します。また `train_data_dir` オプションと `reg_data_dir` オプションを用います。
+
+### step 1. 学習用画像の準備
+
+学習用画像を格納するフォルダを作成します。 __さらにその中に__ 、以下の名前でディレクトリを作成します。
+
+```
+<繰り返し回数>_<identifier> <class>
+```
+
+間の``_``を忘れないでください。
+
+たとえば「sls frog」というプロンプトで、データを20回繰り返す場合、「20_sls frog」となります。以下のようになります。
+
+![image](https://user-images.githubusercontent.com/52813779/210770636-1c851377-5936-4c15-90b7-8ac8ad6c2074.png)
+
+### 複数class、複数対象（identifier）の学習
+
+方法は単純で、学習用画像のフォルダ内に ``繰り返し回数_<identifier> <class>`` のフォルダを複数、正則化画像フォルダにも同様に ``繰り返し回数_<class>`` のフォルダを複数、用意してください。
+
+たとえば「sls frog」と「cpc rabbit」を同時に学習する場合、以下のようになります。
+
+![image](https://user-images.githubusercontent.com/52813779/210777933-a22229db-b219-4cd8-83ca-e87320fc4192.png)
+
+classがひとつで対象が複数の場合、正則化画像フォルダはひとつで構いません。たとえば1girlにキャラAとキャラBがいる場合は次のようにします。
+
+- train_girls
+  - 10_sls 1girl
+  - 10_cpc 1girl
+- reg_girls
+  - 1_1girl
+
+### step 2. 正則化画像の準備
+
+正則化画像を使う場合の手順です。
+
+正則化画像を格納するフォルダを作成します。 __さらにその中に__  ``<繰り返し回数>_<class>`` という名前でディレクトリを作成します。
+
+たとえば「frog」というプロンプトで、データを繰り返さない（1回だけ）場合、以下のようになります。
+
+![image](https://user-images.githubusercontent.com/52813779/210770897-329758e5-3675-49f1-b345-c135f1725832.png)
+
+
+### step 3. 学習の実行
+
+各学習スクリプトを実行します。 `--train_data_dir` オプションで前述の学習用データのフォルダを（__画像を含むフォルダではなく、その親フォルダ__）、`--reg_data_dir` オプションで正則化画像のフォルダ（__画像を含むフォルダではなく、その親フォルダ__）を指定してください。
+
+## DreamBooth、キャプション方式
+
+学習用画像、正則化画像のフォルダに、画像と同じファイル名で、拡張子.caption（オプションで変えられます）のファイルを置くと、そのファイルからキャプションを読み込みプロンプトとして学習します。
+
+※それらの画像の学習に、フォルダ名（identifier class）は使用されなくなります。
+
+キャプションファイルの拡張子はデフォルトで.captionです。学習スクリプトの `--caption_extension` オプションで変更できます。`--shuffle_caption` オプションで学習時のキャプションについて、カンマ区切りの各部分をシャッフルしながら学習します。
+
+## fine tuning 方式
+
+メタデータを作るところまでは設定ファイルを使う場合と同様です。`in_json` オプションでメタデータファイルを指定します。
+
+# 学習途中でのサンプル出力
+
+学習中のモデルで試しに画像生成することで学習の進み方を確認できます。学習スクリプトに以下のオプションを指定します。
+
+- `--sample_every_n_steps` / `--sample_every_n_epochs`
+    
+    サンプル出力するステップ数またはエポック数を指定します。この数ごとにサンプル出力します。両方指定するとエポック数が優先されます。
+
+- `--sample_prompts`
+
+    サンプル出力用プロンプトのファイルを指定します。
+
+- `--sample_sampler`
+
+    サンプル出力に使うサンプラーを指定します。
+    `'ddim', 'pndm', 'heun', 'dpmsolver', 'dpmsolver++', 'dpmsingle', 'k_lms', 'k_euler', 'k_euler_a', 'k_dpm_2', 'k_dpm_2_a'`が選べます。
+
+サンプル出力を行うにはあらかじめプロンプトを記述したテキストファイルを用意しておく必要があります。1行につき1プロンプトで記述します。
+
+たとえば以下のようになります。
+
+```txt
+# prompt 1
+masterpiece, best quality, 1girl, in white shirts, upper body, looking at viewer, simple background --n low quality, worst quality, bad anatomy,bad composition, poor, low effort --w 768 --h 768 --d 1 --l 7.5 --s 28
+
+# prompt 2
+masterpiece, best quality, 1boy, in business suit, standing at street, looking back --n low quality, worst quality, bad anatomy,bad composition, poor, low effort --w 576 --h 832 --d 2 --l 5.5 --s 40
+```
+
+先頭が `#` の行はコメントになります。`--n` のように 「`--` + 英小文字」で生成画像へのオプションを指定できます。以下が使えます。
+
+- `--n` 次のオプションまでをネガティブプロンプトとします。
+- `--w` 生成画像の横幅を指定します。
+- `--h` 生成画像の高さを指定します。
+- `--d` 生成画像のseedを指定します。
+- `--l` 生成画像のCFG scaleを指定します。
+- `--s` 生成時のステップ数を指定します。
+
+
+# 各スクリプトで共通の、よく使われるオプション
+
+スクリプトの更新後、ドキュメントの更新が追い付いていない場合があります。その場合は `--help` オプションで使用できるオプションを確認してください。
+
+## 学習に使うモデル指定
+
+- `--v2` / `--v_parameterization`
+    
+    学習対象モデルとしてHugging Faceのstable-diffusion-2-base、またはそこからのfine tuningモデルを使う場合（推論時に `v2-inference.yaml` を使うように指示されているモデルの場合）は `--v2` オプションを、stable-diffusion-2や768-v-ema.ckpt、およびそれらのfine tuningモデルを使う場合（推論時に `v2-inference-v.yaml` を使うモデルの場合）は `--v2` と `--v_parameterization` の両方のオプションを指定してください。
+
+    Stable Diffusion 2.0では大きく以下の点が変わっています。
+
+    1. 使用するTokenizer
+    2. 使用するText Encoderおよび使用する出力層（2.0は最後から二番目の層を使う）
+    3. Text Encoderの出力次元数（768->1024）
+    4. U-Netの構造（CrossAttentionのhead数など）
+    5. v-parameterization（サンプリング方法が変更されているらしい）
+
+    このうちbaseでは1～4が、baseのつかない方（768-v）では1～5が採用されています。1～4を有効にするのがv2オプション、5を有効にするのがv_parameterizationオプションです。
+
+- `--pretrained_model_name_or_path` 
+    
+    追加学習を行う元となるモデルを指定します。Stable Diffusionのcheckpointファイル（.ckptまたは.safetensors）、Diffusersのローカルディスクにあるモデルディレクトリ、DiffusersのモデルID（"stabilityai/stable-diffusion-2"など）が指定できます。
+
+## 学習に関する設定
+
+- `--output_dir` 
+
+    学習後のモデルを保存するフォルダを指定します。
+    
+- `--output_name` 
+    
+    モデルのファイル名を拡張子を除いて指定します。
+    
+- `--dataset_config` 
+
+    データセットの設定を記述した `.toml` ファイルを指定します。
+
+- `--max_train_steps` / `--max_train_epochs`
+
+    学習するステップ数やエポック数を指定します。両方指定するとエポック数のほうが優先されます。
+
+- `--mixed_precision`
+
+    省メモリ化のため mixed precision （混合精度）で学習します。`--mixed_precision="fp16"` のように指定します。mixed precision なし（デフォルト）と比べて精度が低くなる可能性がありますが、学習に必要なGPUメモリ量が大きく減ります。
+    
+    （RTX30 シリーズ以降では `bf16` も指定できます。環境整備時にaccelerateに行った設定と合わせてください）。
+    
+- `--gradient_checkpointing`
+
+    学習時の重みの計算をまとめて行うのではなく少しずつ行うことで、学習に必要なGPUメモリ量を減らします。オンオフは精度には影響しませんが、オンにするとバッチサイズを大きくできるため、そちらでの影響はあります。
+    
+    また一般的にはオンにすると速度は低下しますが、バッチサイズを大きくできるので、トータルでの学習時間はむしろ速くなるかもしれません。
+
+- `--xformers` / `--mem_eff_attn`
+
+    xformersオプションを指定するとxformersのCrossAttentionを用います。xformersをインストールしていない場合やエラーとなる場合（環境にもよりますが `mixed_precision="no"` の場合など）、代わりに `mem_eff_attn` オプションを指定すると省メモリ版CrossAttentionを使用します（xformersよりも速度は遅くなります）。
+
+- `--clip_skip`
+    
+    `2` を指定すると、Text Encoder (CLIP) の後ろから二番目の層の出力を用います。1またはオプション省略時は最後の層を用います。
+
+    ※SD2.0はデフォルトで後ろから二番目の層を使うため、SD2.0の学習では指定しないでください。
+
+    学習対象のモデルがもともと二番目の層を使うように学習されている場合は、2を指定するとよいでしょう。
+
+    そうではなく最後の層を使用していた場合はモデル全体がそれを前提に学習されています。そのため改めて二番目の層を使用して学習すると、望ましい学習結果を得るにはある程度の枚数の教師データ、長めの学習が必要になるかもしれません。
+
+- `--max_token_length`
+
+    デフォルトは75です。`150` または `225` を指定することでトークン長を拡張して学習できます。長いキャプションで学習する場合に指定してください。
+    
+    ただし学習時のトークン拡張の仕様は Automatic1111 氏のWeb UIとは微妙に異なるため（分割の仕様など）、必要なければ75で学習することをお勧めします。
+
+    clip_skipと同様に、モデルの学習状態と異なる長さで学習するには、ある程度の教師データ枚数、長めの学習時間が必要になると思われます。
+
+- `--weighted_captions`
+
+    指定するとAutomatic1111氏のWeb UIと同様の重み付きキャプションが有効になります。「Textual Inversion と XTI」以外の学習に使用できます。キャプションだけでなく DreamBooth 手法の token string でも有効です。
+
+    重みづけキャプションの記法はWeb UIとほぼ同じで、(abc)や[abc]、(abc:1.23)などが使用できます。入れ子も可能です。括弧内にカンマを含めるとプロンプトのshuffle/dropoutで括弧の対応付けがおかしくなるため、括弧内にはカンマを含めないでください。
+
+- `--persistent_data_loader_workers`
+
+    Windows環境で指定するとエポック間の待ち時間が大幅に短縮されます。
+
+- `--max_data_loader_n_workers`
+
+    データ読み込みのプロセス数を指定します。プロセス数が多いとデータ読み込みが速くなりGPUを効率的に利用できますが、メインメモリを消費します。デフォルトは「`8` または `CPU同時実行スレッド数-1` の小さいほう」なので、メインメモリに余裕がない場合や、GPU使用率が90%程度以上なら、それらの数値を見ながら `2` または `1` 程度まで下げてください。
+
+- `--logging_dir` / `--log_prefix`
+
+    学習ログの保存に関するオプションです。logging_dirオプションにログ保存先フォルダを指定してください。TensorBoard形式のログが保存されます。
+
+    たとえば--logging_dir=logsと指定すると、作業フォルダにlogsフォルダが作成され、その中の日時フォルダにログが保存されます。
+    また--log_prefixオプションを指定すると、日時の前に指定した文字列が追加されます。「--logging_dir=logs --log_prefix=db_style1_」などとして識別用にお使いください。
+
+    TensorBoardでログを確認するには、別のコマンドプロンプトを開き、作業フォルダで以下のように入力します。
+
+    ```
+    tensorboard --logdir=logs
+    ```
+
+    （tensorboardは環境整備時にあわせてインストールされると思いますが、もし入っていないなら `pip install tensorboard` で入れてください。）
+
+    その後ブラウザを開き、http://localhost:6006/ へアクセスすると表示されます。
+
+- `--log_with` / `--log_tracker_name`
+
+    学習ログの保存に関するオプションです。`tensorboard` だけでなく `wandb`への保存が可能です。詳細は [PR#428](https://github.com/kohya-ss/sd-scripts/pull/428)をご覧ください。
+
+- `--noise_offset`
+
+    こちらの記事の実装になります: https://www.crosslabs.org//blog/diffusion-with-offset-noise
+    
+    全体的に暗い、明るい画像の生成結果が良くなる可能性があるようです。LoRA学習でも有効なようです。`0.1` 程度の値を指定するとよいようです。
+
+- `--adaptive_noise_scale` （実験的オプション）
+
+    Noise offsetの値を、latentsの各チャネルの平均値の絶対値に応じて自動調整するオプションです。`--noise_offset` と同時に指定することで有効になります。Noise offsetの値は `noise_offset + abs(mean(latents, dim=(2,3))) * adaptive_noise_scale` で計算されます。latentは正規分布に近いためnoise_offsetの1/10～同程度の値を指定するとよいかもしれません。
+
+    負の値も指定でき、その場合はnoise offsetは0以上にclipされます。
+
+- `--multires_noise_iterations` / `--multires_noise_discount`
+    
+    Multi resolution noise (pyramid noise)の設定です。詳細は [PR#471](https://github.com/kohya-ss/sd-scripts/pull/471) およびこちらのページ [Multi-Resolution Noise for Diffusion Model Training](https://wandb.ai/johnowhitaker/multires_noise/reports/Multi-Resolution-Noise-for-Diffusion-Model-Training--VmlldzozNjYyOTU2) を参照してください。
+    
+    `--multires_noise_iterations` に数値を指定すると有効になります。6~10程度の値が良いようです。`--multires_noise_discount` に0.1~0.3 程度の値（LoRA学習等比較的データセットが小さい場合のPR作者の推奨）、ないしは0.8程度の値（元記事の推奨）を指定してください（デフォルトは 0.3）。
+
+- `--debug_dataset`
+
+    このオプションを付けることで学習を行う前に事前にどのような画像データ、キャプションで学習されるかを確認できます。Escキーを押すと終了してコマンドラインに戻ります。`S`キーで次のステップ（バッチ）、`E`キーで次のエポックに進みます。
+
+    ※Linux環境（Colabを含む）では画像は表示されません。
+
+- `--vae`
+
+    vaeオプションにStable Diffusionのcheckpoint、VAEのcheckpointファイル、DiffusesのモデルまたはVAE（ともにローカルまたはHugging FaceのモデルIDが指定できます）のいずれかを指定すると、そのVAEを使って学習します（latentsのキャッシュ時または学習中のlatents取得時）。
+
+    DreamBoothおよびfine tuningでは、保存されるモデルはこのVAEを組み込んだものになります。
+
+- `--cache_latents` / `--cache_latents_to_disk`
+
+    使用VRAMを減らすためVAEの出力をメインメモリにキャッシュします。`flip_aug` 以外のaugmentationは使えなくなります。また全体の学習速度が若干速くなります。
+
+    cache_latents_to_diskを指定するとキャッシュをディスクに保存します。スクリプトを終了し、再度起動した場合もキャッシュが有効になります。
+
+- `--min_snr_gamma`
+
+    Min-SNR Weighting strategyを指定します。詳細は[こちら](https://github.com/kohya-ss/sd-scripts/pull/308)を参照してください。論文では`5`が推奨されています。
+
+## モデルの保存に関する設定
+
+- `--save_precision`
+
+    保存時のデータ精度を指定します。save_precisionオプションにfloat、fp16、bf16のいずれかを指定すると、その形式でモデルを保存します（DreamBooth、fine tuningでDiffusers形式でモデルを保存する場合は無効です）。モデルのサイズを削減したい場合などにお使いください。
+
+- `--save_every_n_epochs` / `--save_state` / `--resume`
+
+    save_every_n_epochsオプションに数値を指定すると、そのエポックごとに学習途中のモデルを保存します。
+
+    save_stateオプションを同時に指定すると、optimizer等の状態も含めた学習状態を合わせて保存します（保存したモデルからも学習再開できますが、それに比べると精度の向上、学習時間の短縮が期待できます）。保存先はフォルダになります。
+    
+    学習状態は保存先フォルダに `<output_name>-??????-state`（??????はエポック数）という名前のフォルダで出力されます。長時間にわたる学習時にご利用ください。
+
+    保存された学習状態から学習を再開するにはresumeオプションを使います。学習状態のフォルダ（`output_dir` ではなくその中のstateのフォルダ）を指定してください。
+
+    なおAcceleratorの仕様により、エポック数、global stepは保存されておらず、resumeしたときにも1からになりますがご容赦ください。
+
+- `--save_every_n_steps`
+
+    save_every_n_stepsオプションに数値を指定すると、そのステップごとに学習途中のモデルを保存します。save_every_n_epochsと同時に指定できます。
+
+- `--save_model_as` （DreamBooth, fine tuning のみ）
+
+    モデルの保存形式を`ckpt, safetensors, diffusers, diffusers_safetensors` から選べます。
+    
+    `--save_model_as=safetensors` のように指定します。Stable Diffusion形式（ckptまたはsafetensors）を読み込み、Diffusers形式で保存する場合、不足する情報はHugging Faceからv1.5またはv2.1の情報を落としてきて補完します。
+
+- `--huggingface_repo_id` 等
+
+    huggingface_repo_idが指定されているとモデル保存時に同時にHuggingFaceにアップロードします。アクセストークンの取り扱いに注意してください（HuggingFaceのドキュメントを参照してください）。
+
+    他の引数をたとえば以下のように指定してください。
+
+    -   `--huggingface_repo_id "your-hf-name/your-model" --huggingface_path_in_repo "path" --huggingface_repo_type model --huggingface_repo_visibility private --huggingface_token hf_YourAccessTokenHere`
+
+    huggingface_repo_visibilityに`public`を指定するとリポジトリが公開されます。省略時または`private`（などpublic以外）を指定すると非公開になります。
+
+    `--save_state`オプション指定時に`--save_state_to_huggingface`を指定するとstateもアップロードします。
+
+    `--resume`オプション指定時に`--resume_from_huggingface`を指定するとHuggingFaceからstateをダウンロードして再開します。その時の --resumeオプションは `--resume {repo_id}/{path_in_repo}:{revision}:{repo_type}`になります。
+    
+    例: `--resume_from_huggingface --resume your-hf-name/your-model/path/test-000002-state:main:model`
+
+    `--async_upload`オプションを指定するとアップロードを非同期で行います。
+
+## オプティマイザ関係
+
+- `--optimizer_type`
+    --オプティマイザの種類を指定します。以下が指定できます。
+    - AdamW : [torch.optim.AdamW](https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html)
+    - 過去のバージョンのオプション未指定時と同じ
+    - AdamW8bit : 引数は同上
+    - 過去のバージョンの--use_8bit_adam指定時と同じ
+    - Lion : https://github.com/lucidrains/lion-pytorch
+    - 過去のバージョンの--use_lion_optimizer指定時と同じ
+    - Lion8bit : 引数は同上
+    - SGDNesterov : [torch.optim.SGD](https://pytorch.org/docs/stable/generated/torch.optim.SGD.html), nesterov=True
+    - SGDNesterov8bit : 引数は同上
+    - DAdaptation(DAdaptAdamPreprint) : https://github.com/facebookresearch/dadaptation
+    - DAdaptAdam : 引数は同上
+    - DAdaptAdaGrad : 引数は同上
+    - DAdaptAdan : 引数は同上
+    - DAdaptAdanIP : 引数は同上
+    - DAdaptLion : 引数は同上
+    - DAdaptSGD : 引数は同上
+    - Prodigy : https://github.com/konstmish/prodigy
+    - AdaFactor : [Transformers AdaFactor](https://huggingface.co/docs/transformers/main_classes/optimizer_schedules)
+    - 任意のオプティマイザ
+
+- `--learning_rate`
+
+    学習率を指定します。適切な学習率は学習スクリプトにより異なりますので、それぞれの説明を参照してください。
+
+- `--lr_scheduler` / `--lr_warmup_steps` / `--lr_scheduler_num_cycles` / `--lr_scheduler_power`
+  
+    学習率のスケジューラ関連の指定です。
+
+    lr_schedulerオプションで学習率のスケジューラをlinear, cosine, cosine_with_restarts, polynomial, constant, constant_with_warmup, 任意のスケジューラから選べます。デフォルトはconstantです。
+    
+    lr_warmup_stepsでスケジューラのウォームアップ（だんだん学習率を変えていく）ステップ数を指定できます。
+    
+    lr_scheduler_num_cycles は cosine with restartsスケジューラでのリスタート回数、lr_scheduler_power は polynomialスケジューラでのpolynomial power です。
+
+    詳細については各自お調べください。
+
+    任意のスケジューラを使う場合、任意のオプティマイザと同様に、`--scheduler_args`でオプション引数を指定してください。
+
+### オプティマイザの指定について
+
+オプティマイザのオプション引数は--optimizer_argsオプションで指定してください。key=valueの形式で、複数の値が指定できます。また、valueはカンマ区切りで複数の値が指定できます。たとえばAdamWオプティマイザに引数を指定する場合は、``--optimizer_args weight_decay=0.01 betas=.9,.999``のようになります。
+
+オプション引数を指定する場合は、それぞれのオプティマイザの仕様をご確認ください。
+
+一部のオプティマイザでは必須の引数があり、省略すると自動的に追加されます（SGDNesterovのmomentumなど）。コンソールの出力を確認してください。
+
+D-Adaptationオプティマイザは学習率を自動調整します。学習率のオプションに指定した値は学習率そのものではなくD-Adaptationが決定した学習率の適用率になりますので、通常は1.0を指定してください。Text EncoderにU-Netの半分の学習率を指定したい場合は、``--text_encoder_lr=0.5 --unet_lr=1.0``と指定します。
+
+AdaFactorオプティマイザはrelative_step=Trueを指定すると学習率を自動調整できます（省略時はデフォルトで追加されます）。自動調整する場合は学習率のスケジューラにはadafactor_schedulerが強制的に使用されます。またscale_parameterとwarmup_initを指定するとよいようです。
+
+自動調整する場合のオプション指定はたとえば ``--optimizer_args "relative_step=True" "scale_parameter=True" "warmup_init=True"`` のようになります。
+
+学習率を自動調整しない場合はオプション引数 ``relative_step=False`` を追加してください。その場合、学習率のスケジューラにはconstant_with_warmupが、また勾配のclip normをしないことが推奨されているようです。そのため引数は ``--optimizer_type=adafactor --optimizer_args "relative_step=False" --lr_scheduler="constant_with_warmup" --max_grad_norm=0.0`` のようになります。
+
+### 任意のオプティマイザを使う
+
+``torch.optim`` のオプティマイザを使う場合にはクラス名のみを（``--optimizer_type=RMSprop``など）、他のモジュールのオプティマイザを使う時は「モジュール名.クラス名」を指定してください（``--optimizer_type=bitsandbytes.optim.lamb.LAMB``など）。
+
+（内部でimportlibしているだけで動作は未確認です。必要ならパッケージをインストールしてください。）
+
+
+<!-- 
+## 任意サイズの画像での学習 --resolution
+正方形以外で学習できます。resolutionに「448,640」のように「幅,高さ」で指定してください。幅と高さは64で割り切れる必要があります。学習用画像、正則化画像のサイズを合わせてください。
+
+個人的には縦長の画像を生成することが多いため「448,640」などで学習することもあります。
+
+## Aspect Ratio Bucketing --enable_bucket / --min_bucket_reso / --max_bucket_reso
+enable_bucketオプションを指定すると有効になります。Stable Diffusionは512x512で学習されていますが、それに加えて256x768や384x640といった解像度でも学習します。
+
+このオプションを指定した場合は、学習用画像、正則化画像を特定の解像度に統一する必要はありません。いくつかの解像度（アスペクト比）から最適なものを選び、その解像度で学習します。
+解像度は64ピクセル単位のため、元画像とアスペクト比が完全に一致しない場合がありますが、その場合は、はみ出した部分がわずかにトリミングされます。
+
+解像度の最小サイズをmin_bucket_resoオプションで、最大サイズをmax_bucket_resoで指定できます。デフォルトはそれぞれ256、1024です。
+たとえば最小サイズに384を指定すると、256x1024や320x768などの解像度は使わなくなります。
+解像度を768x768のように大きくした場合、最大サイズに1280などを指定しても良いかもしれません。
+
+なおAspect Ratio Bucketingを有効にするときには、正則化画像についても、学習用画像と似た傾向の様々な解像度を用意した方がいいかもしれません。
+
+（ひとつのバッチ内の画像が学習用画像、正則化画像に偏らなくなるため。そこまで大きな影響はないと思いますが……。）
+
+## augmentation --color_aug / --flip_aug
+augmentationは学習時に動的にデータを変化させることで、モデルの性能を上げる手法です。color_augで色合いを微妙に変えつつ、flip_augで左右反転をしつつ、学習します。
+
+動的にデータを変化させるため、cache_latentsオプションと同時に指定できません。
+
+
+## 勾配をfp16とした学習（実験的機能） --full_fp16
+full_fp16オプションを指定すると勾配を通常のfloat32からfloat16（fp16）に変更して学習します（mixed precisionではなく完全なfp16学習になるようです）。
+これによりSD1.xの512x512サイズでは8GB未満、SD2.xの512x512サイズで12GB未満のVRAM使用量で学習できるようです。
+
+あらかじめaccelerate configでfp16を指定し、オプションで ``mixed_precision="fp16"`` としてください（bf16では動作しません）。
+
+メモリ使用量を最小化するためには、xformers、use_8bit_adam、cache_latents、gradient_checkpointingの各オプションを指定し、train_batch_sizeを1としてください。
+
+（余裕があるようならtrain_batch_sizeを段階的に増やすと若干精度が上がるはずです。）
+
+PyTorchのソースにパッチを当てて無理やり実現しています（PyTorch 1.12.1と1.13.0で確認）。精度はかなり落ちますし、途中で学習失敗する確率も高くなります。
+学習率やステップ数の設定もシビアなようです。それらを認識したうえで自己責任でお使いください。
+
+-->
+
+# メタデータファイルの作成
+
+## 教師データの用意
+
+前述のように学習させたい画像データを用意し、任意のフォルダに入れてください。
+
+たとえば以下のように画像を格納します。
+
+![教師データフォルダのスクショ](https://user-images.githubusercontent.com/52813779/208907739-8e89d5fa-6ca8-4b60-8927-f484d2a9ae04.png)
+
+## 自動キャプショニング
+
+キャプションを使わずタグだけで学習する場合はスキップしてください。
+
+また手動でキャプションを用意する場合、キャプションは教師データ画像と同じディレクトリに、同じファイル名、拡張子.caption等で用意してください。各ファイルは1行のみのテキストファイルとします。
+
+### BLIPによるキャプショニング
+
+最新版ではBLIPのダウンロード、重みのダウンロード、仮想環境の追加は不要になりました。そのままで動作します。
+
+finetuneフォルダ内のmake_captions.pyを実行します。
+
+```
+python finetune\make_captions.py --batch_size <バッチサイズ> <教師データフォルダ>
+```
+
+バッチサイズ8、教師データを親フォルダのtrain_dataに置いた場合、以下のようになります。
+
+```
+python finetune\make_captions.py --batch_size 8 ..\train_data
+```
+
+キャプションファイルが教師データ画像と同じディレクトリに、同じファイル名、拡張子.captionで作成されます。
+
+batch_sizeはGPUのVRAM容量に応じて増減してください。大きいほうが速くなります（VRAM 12GBでももう少し増やせると思います）。
+max_lengthオプションでキャプションの最大長を指定できます。デフォルトは75です。モデルをトークン長225で学習する場合には長くしても良いかもしれません。
+caption_extensionオプションでキャプションの拡張子を変更できます。デフォルトは.captionです（.txtにすると後述のDeepDanbooruと競合します）。
+
+複数の教師データフォルダがある場合には、それぞれのフォルダに対して実行してください。
+
+なお、推論にランダム性があるため、実行するたびに結果が変わります。固定する場合には--seedオプションで `--seed 42` のように乱数seedを指定してください。
+
+その他のオプションは `--help` でヘルプをご参照ください（パラメータの意味についてはドキュメントがまとまっていないようで、ソースを見るしかないようです）。
+
+デフォルトでは拡張子.captionでキャプションファイルが生成されます。
+
+![captionが生成されたフォルダ](https://user-images.githubusercontent.com/52813779/208908845-48a9d36c-f6ee-4dae-af71-9ab462d1459e.png)
+
+たとえば以下のようなキャプションが付きます。
+
+![キャプションと画像](https://user-images.githubusercontent.com/52813779/208908947-af936957-5d73-4339-b6c8-945a52857373.png)
+
+## DeepDanbooruによるタグ付け
+
+danbooruタグのタグ付け自体を行わない場合は「キャプションとタグ情報の前処理」に進んでください。
+
+タグ付けはDeepDanbooruまたはWD14Taggerで行います。WD14Taggerのほうが精度が良いようです。WD14Taggerでタグ付けする場合は、次の章へ進んでください。
+
+### 環境整備
+
+DeepDanbooru https://github.com/KichangKim/DeepDanbooru  を作業フォルダにcloneしてくるか、zipをダウンロードして展開します。私はzipで展開しました。
+またDeepDanbooruのReleasesのページ https://github.com/KichangKim/DeepDanbooru/releases  の「DeepDanbooru Pretrained Model v3-20211112-sgd-e28」のAssetsから、deepdanbooru-v3-20211112-sgd-e28.zipをダウンロードしてきてDeepDanbooruのフォルダに展開します。
+
+以下からダウンロードします。Assetsをクリックして開き、そこからダウンロードします。
+
+![DeepDanbooruダウンロードページ](https://user-images.githubusercontent.com/52813779/208909417-10e597df-7085-41ee-bd06-3e856a1339df.png)
+
+以下のようなこういうディレクトリ構造にしてください
+
+![DeepDanbooruのディレクトリ構造](https://user-images.githubusercontent.com/52813779/208909486-38935d8b-8dc6-43f1-84d3-fef99bc471aa.png)
+
+Diffusersの環境に必要なライブラリをインストールします。DeepDanbooruのフォルダに移動してインストールします（実質的にはtensorflow-ioが追加されるだけだと思います）。
+
+```
+pip install -r requirements.txt
+```
+
+続いてDeepDanbooru自体をインストールします。
+
+```
+pip install .
+```
+
+以上でタグ付けの環境整備は完了です。
+
+### タグ付けの実施
+DeepDanbooruのフォルダに移動し、deepdanbooruを実行してタグ付けを行います。
+
+```
+deepdanbooru evaluate <教師データフォルダ> --project-path deepdanbooru-v3-20211112-sgd-e28 --allow-folder --save-txt
+```
+
+教師データを親フォルダのtrain_dataに置いた場合、以下のようになります。
+
+```
+deepdanbooru evaluate ../train_data --project-path deepdanbooru-v3-20211112-sgd-e28 --allow-folder --save-txt
+```
+
+タグファイルが教師データ画像と同じディレクトリに、同じファイル名、拡張子.txtで作成されます。1件ずつ処理されるためわりと遅いです。
+
+複数の教師データフォルダがある場合には、それぞれのフォルダに対して実行してください。
+
+以下のように生成されます。
+
+![DeepDanbooruの生成ファイル](https://user-images.githubusercontent.com/52813779/208909855-d21b9c98-f2d3-4283-8238-5b0e5aad6691.png)
+
+こんな感じにタグが付きます（すごい情報量……）。
+
+![DeepDanbooruタグと画像](https://user-images.githubusercontent.com/52813779/208909908-a7920174-266e-48d5-aaef-940aba709519.png)
+
+## WD14Taggerによるタグ付け
+
+DeepDanbooruの代わりにWD14Taggerを用いる手順です。
+
+Automatic1111氏のWebUIで使用しているtaggerを利用します。こちらのgithubページ（https://github.com/toriato/stable-diffusion-webui-wd14-tagger#mrsmilingwolfs-model-aka-waifu-diffusion-14-tagger ）の情報を参考にさせていただきました。
+
+最初の環境整備で必要なモジュールはインストール済みです。また重みはHugging Faceから自動的にダウンロードしてきます。
+
+### タグ付けの実施
+
+スクリプトを実行してタグ付けを行います。
+```
+python tag_images_by_wd14_tagger.py --batch_size <バッチサイズ> <教師データフォルダ>
+```
+
+教師データを親フォルダのtrain_dataに置いた場合、以下のようになります。
+```
+python tag_images_by_wd14_tagger.py --batch_size 4 ..\train_data
+```
+
+初回起動時にはモデルファイルがwd14_tagger_modelフォルダに自動的にダウンロードされます（フォルダはオプションで変えられます）。以下のようになります。
+
+![ダウンロードされたファイル](https://user-images.githubusercontent.com/52813779/208910447-f7eb0582-90d6-49d3-a666-2b508c7d1842.png)
+
+タグファイルが教師データ画像と同じディレクトリに、同じファイル名、拡張子.txtで作成されます。
+
+![生成されたタグファイル](https://user-images.githubusercontent.com/52813779/208910534-ea514373-1185-4b7d-9ae3-61eb50bc294e.png)
+
+![タグと画像](https://user-images.githubusercontent.com/52813779/208910599-29070c15-7639-474f-b3e4-06bd5a3df29e.png)
+
+threshオプションで、判定されたタグのconfidence（確信度）がいくつ以上でタグをつけるかが指定できます。デフォルトはWD14Taggerのサンプルと同じ0.35です。値を下げるとより多くのタグが付与されますが、精度は下がります。
+
+batch_sizeはGPUのVRAM容量に応じて増減してください。大きいほうが速くなります（VRAM 12GBでももう少し増やせると思います）。caption_extensionオプションでタグファイルの拡張子を変更できます。デフォルトは.txtです。
+
+model_dirオプションでモデルの保存先フォルダを指定できます。
+
+またforce_downloadオプションを指定すると保存先フォルダがあってもモデルを再ダウンロードします。
+
+複数の教師データフォルダがある場合には、それぞれのフォルダに対して実行してください。
+
+## キャプションとタグ情報の前処理
+
+スクリプトから処理しやすいようにキャプションとタグをメタデータとしてひとつのファイルにまとめます。
+
+### キャプションの前処理
+
+キャプションをメタデータに入れるには、作業フォルダ内で以下を実行してください（キャプションを学習に使わない場合は実行不要です）（実際は1行で記述します、以下同様）。`--full_path` オプションを指定してメタデータに画像ファイルの場所をフルパスで格納します。このオプションを省略すると相対パスで記録されますが、フォルダ指定が `.toml` ファイル内で別途必要になります。
+
+```
+python merge_captions_to_metadata.py --full_path <教師データフォルダ>
+　  --in_json <読み込むメタデータファイル名> <メタデータファイル名>
+```
+
+メタデータファイル名は任意の名前です。
+教師データがtrain_data、読み込むメタデータファイルなし、メタデータファイルがmeta_cap.jsonの場合、以下のようになります。
+
+```
+python merge_captions_to_metadata.py --full_path train_data meta_cap.json
+```
+
+caption_extensionオプションでキャプションの拡張子を指定できます。
+
+複数の教師データフォルダがある場合には、full_path引数を指定しつつ、それぞれのフォルダに対して実行してください。
+
+```
+python merge_captions_to_metadata.py --full_path 
+    train_data1 meta_cap1.json
+python merge_captions_to_metadata.py --full_path --in_json meta_cap1.json 
+    train_data2 meta_cap2.json
+```
+
+in_jsonを省略すると書き込み先メタデータファイルがあるとそこから読み込み、そこに上書きします。
+
+__※in_jsonオプションと書き込み先を都度書き換えて、別のメタデータファイルへ書き出すようにすると安全です。__
+
+### タグの前処理
+
+同様にタグもメタデータにまとめます（タグを学習に使わない場合は実行不要です）。
+```
+python merge_dd_tags_to_metadata.py --full_path <教師データフォルダ> 
+    --in_json <読み込むメタデータファイル名> <書き込むメタデータファイル名>
+```
+
+先と同じディレクトリ構成で、meta_cap.jsonを読み、meta_cap_dd.jsonに書きだす場合、以下となります。
+```
+python merge_dd_tags_to_metadata.py --full_path train_data --in_json meta_cap.json meta_cap_dd.json
+```
+
+複数の教師データフォルダがある場合には、full_path引数を指定しつつ、それぞれのフォルダに対して実行してください。
+
+```
+python merge_dd_tags_to_metadata.py --full_path --in_json meta_cap2.json
+    train_data1 meta_cap_dd1.json
+python merge_dd_tags_to_metadata.py --full_path --in_json meta_cap_dd1.json 
+    train_data2 meta_cap_dd2.json
+```
+
+in_jsonを省略すると書き込み先メタデータファイルがあるとそこから読み込み、そこに上書きします。
+
+__※in_jsonオプションと書き込み先を都度書き換えて、別のメタデータファイルへ書き出すようにすると安全です。__
+
+### キャプションとタグのクリーニング
+
+ここまででメタデータファイルにキャプションとDeepDanbooruのタグがまとめられています。ただ自動キャプショニングにしたキャプションは表記ゆれなどがあり微妙（※）ですし、タグにはアンダースコアが含まれていたりratingが付いていたりしますので（DeepDanbooruの場合）、エディタの置換機能などを用いてキャプションとタグのクリーニングをしたほうがいいでしょう。
+
+※たとえばアニメ絵の少女を学習する場合、キャプションにはgirl/girls/woman/womenなどのばらつきがあります。また「anime girl」なども単に「girl」としたほうが適切かもしれません。
+
+クリーニング用のスクリプトが用意してありますので、スクリプトの内容を状況に応じて編集してお使いください。
+
+（教師データフォルダの指定は不要になりました。メタデータ内の全データをクリーニングします。）
+
+```
+python clean_captions_and_tags.py <読み込むメタデータファイル名> <書き込むメタデータファイル名>
+```
+
+--in_jsonは付きませんのでご注意ください。たとえば次のようになります。
+
+```
+python clean_captions_and_tags.py meta_cap_dd.json meta_clean.json
+```
+
+以上でキャプションとタグの前処理は完了です。
+
+## latentsの事前取得
+
+※ このステップは必須ではありません。省略しても学習時にlatentsを取得しながら学習できます。
+また学習時に `random_crop` や `color_aug` などを行う場合にはlatentsの事前取得はできません（画像を毎回変えながら学習するため）。事前取得をしない場合、ここまでのメタデータで学習できます。
+
+あらかじめ画像の潜在表現を取得しディスクに保存しておきます。それにより、学習を高速に進めることができます。あわせてbucketing（教師データをアスペクト比に応じて分類する）を行います。
+
+作業フォルダで以下のように入力してください。
+```
+python prepare_buckets_latents.py --full_path <教師データフォルダ>  
+    <読み込むメタデータファイル名> <書き込むメタデータファイル名> 
+    <fine tuningするモデル名またはcheckpoint> 
+    --batch_size <バッチサイズ> 
+    --max_resolution <解像度 幅,高さ> 
+    --mixed_precision <精度>
+```
+
+モデルがmodel.ckpt、バッチサイズ4、学習解像度は512\*512、精度no（float32）で、meta_clean.jsonからメタデータを読み込み、meta_lat.jsonに書き込む場合、以下のようになります。
+
+```
+python prepare_buckets_latents.py --full_path 
+    train_data meta_clean.json meta_lat.json model.ckpt 
+    --batch_size 4 --max_resolution 512,512 --mixed_precision no
+```
+
+教師データフォルダにnumpyのnpz形式でlatentsが保存されます。
+
+解像度の最小サイズを--min_bucket_resoオプションで、最大サイズを--max_bucket_resoで指定できます。デフォルトはそれぞれ256、1024です。たとえば最小サイズに384を指定すると、256\*1024や320\*768などの解像度は使わなくなります。
+解像度を768\*768のように大きくした場合、最大サイズに1280などを指定すると良いでしょう。
+
+--flip_augオプションを指定すると左右反転のaugmentation（データ拡張）を行います。疑似的にデータ量を二倍に増やすことができますが、データが左右対称でない場合に指定すると（例えばキャラクタの外見、髪型など）学習がうまく行かなくなります。
+
+
+（反転した画像についてもlatentsを取得し、\*\_flip.npzファイルを保存する単純な実装です。fline_tune.pyには特にオプション指定は必要ありません。\_flip付きのファイルがある場合、flip付き・なしのファイルを、ランダムに読み込みます。）
+
+バッチサイズはVRAM 12GBでももう少し増やせるかもしれません。
+解像度は64で割り切れる数字で、"幅,高さ"で指定します。解像度はfine tuning時のメモリサイズに直結します。VRAM 12GBでは512,512が限界と思われます（※）。16GBなら512,704や512,768まで上げられるかもしれません。なお256,256等にしてもVRAM 8GBでは厳しいようです（パラメータやoptimizerなどは解像度に関係せず一定のメモリが必要なため）。
+
+※batch size 1の学習で12GB VRAM、640,640で動いたとの報告もありました。
+
+以下のようにbucketingの結果が表示されます。
+
+![bucketingの結果](https://user-images.githubusercontent.com/52813779/208911419-71c00fbb-2ce6-49d5-89b5-b78d7715e441.png)
+
+複数の教師データフォルダがある場合には、full_path引数を指定しつつ、それぞれのフォルダに対して実行してください。
+```
+python prepare_buckets_latents.py --full_path  
+    train_data1 meta_clean.json meta_lat1.json model.ckpt 
+    --batch_size 4 --max_resolution 512,512 --mixed_precision no
+
+python prepare_buckets_latents.py --full_path 
+    train_data2 meta_lat1.json meta_lat2.json model.ckpt 
+    --batch_size 4 --max_resolution 512,512 --mixed_precision no
+
+```
+読み込み元と書き込み先を同じにすることも可能ですが別々の方が安全です。
+
+__※引数を都度書き換えて、別のメタデータファイルに書き込むと安全です。__
+
diff --git a/docs/train_README-zh.md b/docs/train_README-zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..454d545616cdd05742af61ff3fc7decb5e5a38f8
--- /dev/null
+++ b/docs/train_README-zh.md
@@ -0,0 +1,907 @@
+__由于文档正在更新中，描述可能有错误。__
+
+# 关于本学习文档，通用描述
+本库支持模型微调(fine tuning)、DreamBooth、训练LoRA和文本反转(Textual Inversion)（包括[XTI:P+](https://github.com/kohya-ss/sd-scripts/pull/327)
+）
+本文档将说明它们通用的学习数据准备方法和选项等。
+
+# 概要
+
+请提前参考本仓库的README，准备好环境。
+
+
+以下本节说明。
+
+1. 关于准备学习数据的新形式（使用设置文件）
+1. 对于在学习中使用的术语的简要解释
+1. 先前的指定格式（不使用设置文件，而是从命令行指定）
+1. 生成学习过程中的示例图像
+1. 各脚本中常用的共同选项
+1. 准备 fine tuning 方法的元数据：如说明文字(打标签)等
+
+
+1. 如果只执行一次，学习就可以进行（相关内容，请参阅各个脚本的文档）。如果需要，以后可以随时参考。
+
+
+
+# 关于准备训练数据
+
+在任意文件夹（也可以是多个文件夹）中准备好训练数据的图像文件。支持 `.png`, `.jpg`, `.jpeg`, `.webp`, `.bmp` 格式的文件。通常不需要进行任何预处理，如调整大小等。
+
+但是请勿使用极小的图像，其尺寸比训练分辨率（稍后将提到）还小，建议事先使用超分辨率AI等进行放大。另外，请注意不要使用过大的图像（约为3000 x 3000像素以上），因为这可能会导致错误，建议事先缩小。
+
+在训练时，需要整理要用于训练模型的图像数据，并将其指定给脚本。根据训练数据的数量、训练目标和说明（图像描述）是否可用等因素，可以使用几种方法指定训练数据。以下是其中的一些方法（每个名称都不是通用的，而是该存储库自定义的定义）。有关正则化图像的信息将在稍后提供。
+
+1. DreamBooth、class + identifier方式（可使用正则化图像）
+
+    将训练目标与特定单词（identifier）相关联进行训练。无需准备说明。例如，当要学习特定角色时，由于无需准备说明，因此比较方便，但由于学习数据的所有元素都与identifier相关联，例如发型、服装、背景等，因此在生成时可能会出现无法更换服装的情况。
+
+2. DreamBooth、说明方式（可使用正则化图像）
+
+    准备记录每个图像说明的文本文件进行训练。例如，通过将图像详细信息（如穿着白色衣服的角色A、穿着红色衣服的角色A等）记录在说明中，可以将角色和其他元素分离，并期望模型更准确地学习角色。
+
+3. 微调方式（不可使用正则化图像）
+
+    先将说明收集到元数据文件中。支持分离标签和说明以及预先缓存latents等功能，以加速训练（这些将在另一篇文档中介绍）。（虽然名为fine tuning方式，但不仅限于fine tuning。）
+你要学的东西和你可以使用的规范方法的组合如下。
+
+| 学习对象或方法        | 脚本 | DB/class+identifier | DB/caption | fine tuning |
+|----------------| ----- | ----- | ----- | ----- |
+| fine tuning微调模型           | `fine_tune.py`| x | x | o |
+| DreamBooth训练模型 | `train_db.py`| o | o | x |
+| LoRA           | `train_network.py`| o | o | o |
+| Textual Invesion | `train_textual_inversion.py`| o | o | o |
+
+## 选择哪一个
+
+如果您想要学习LoRA、Textual Inversion而不需要准备简介文件，则建议使用DreamBooth class+identifier。如果您能够准备好，则DreamBooth Captions方法更好。如果您有大量的训练数据并且不使用规则化图像，则请考虑使用fine-tuning方法。
+
+对于DreamBooth也是一样的，但不能使用fine-tuning方法。对于fine-tuning方法，只能使用fine-tuning方式。
+
+# 每种方法的指定方式
+
+在这里，我们只介绍每种指定方法的典型模式。有关更详细的指定方法，请参见[数据集设置](./config_README-ja.md)。
+
+# DreamBooth，class+identifier方法（可使用规则化图像）
+
+在该方法中，每个图像将被视为使用与 `class identifier` 相同的标题进行训练（例如 `shs dog`）。
+
+这样一来，每张图片都相当于使用标题“分类标识”（例如“shs dog”）进行训练。
+
+## step 1.确定identifier和class
+
+要将学习的目标与identifier和属于该目标的class相关联。
+
+（虽然有很多称呼，但暂时按照原始论文的说法。）
+
+以下是简要说明（请查阅详细信息）。
+
+class是学习目标的一般类别。例如，如果要学习特定品种的狗，则class将是“dog”。对于动漫角色，根据模型不同，可能是“boy”或“girl”，也可能是“1boy”或“1girl”。
+
+identifier是用于识别学习目标并进行学习的单词。可以使用任何单词，但是根据原始论文，“Tokenizer生成的3个或更少字符的罕见单词”是最好的选择。
+
+使用identifier和class，例如，“shs dog”可以将模型训练为从class中识别并学习所需的目标。
+
+在图像生成时，使用“shs dog”将生成所学习狗种的图像。
+
+（作为identifier，我最近使用的一些参考是“shs sts scs cpc coc cic msm usu ici lvl cic dii muk ori hru rik koo yos wny”等。最好是不包含在Danbooru标签中的单词。）
+
+## step 2. 决定是否使用正则化图像，并生成正则化图像
+
+正则化图像是为防止前面提到的语言漂移，即整个类别被拉扯成为学习目标而生成的图像。如果不使用正则化图像，例如在 `shs 1girl` 中学习特定角色时，即使在简单的 `1girl` 提示下生成，也会越来越像该角色。这是因为 `1girl` 在训练时的标题中包含了该角色的信息。
+
+通过同时学习目标图像和正则化图像，类别仍然保持不变，仅在将标识符附加到提示中时才生成目标图像。
+
+如果您只想在LoRA或DreamBooth中使用特定的角色，则可以不使用正则化图像。
+
+在Textual Inversion中也不需要使用（如果要学习的token string不包含在标题中，则不会学习任何内容）。
+
+一般情况下，使用在训练目标模型时只使用类别名称生成的图像作为正则化图像是常见的做法（例如 `1girl`）。但是，如果生成的图像质量不佳，可以尝试修改提示或使用从网络上另外下载的图像。
+
+（由于正则化图像也被训练，因此其质量会影响模型。）
+
+通常，准备数百张图像是理想的（图像数量太少会导致类别图像无法推广并学习它们的特征）。
+
+如果要使用生成的图像，请将其大小通常与训练分辨率（更准确地说是bucket的分辨率）相适应。
+
+## step 2. 设置文件的描述
+
+创建一个文本文件，并将其扩展名更改为`.toml`。例如，您可以按以下方式进行描述：
+
+（以`＃`开头的部分是注释，因此您可以直接复制粘贴，或者将其删除，都没有问题。）
+
+```toml
+[general]
+enable_bucket = true                        # 是否使用Aspect Ratio Bucketing
+
+[[datasets]]
+resolution = 512                            # 学习分辨率
+batch_size = 4                              # 批量大小
+
+  [[datasets.subsets]]
+  image_dir = 'C:\hoge'                     # 指定包含训练图像的文件夹
+  class_tokens = 'hoge girl'                # 指定标识符类
+  num_repeats = 10                          # 训练图像的迭代次数
+
+  # 以下仅在使用正则化图像时进行描述。不使用则删除
+  [[datasets.subsets]]
+  is_reg = true
+  image_dir = 'C:\reg'                      # 指定包含正则化图像的文件夹
+  class_tokens = 'girl'                     # 指定类别
+  num_repeats = 1                           # 正则化图像的迭代次数，基本上1就可以了
+```
+
+基本上只需更改以下位置即可进行学习。
+
+1. 学习分辨率
+
+    指定一个数字表示正方形（如果是 `512`，则为 512x512），如果使用方括号和逗号分隔的两个数字，则表示横向×纵向（如果是`[512,768]`，则为 512x768）。在SD1.x系列中，原始学习分辨率为512。指定较大的分辨率，如 `[512,768]` 可能会减少纵向和横向图像生成时的错误。在SD2.x 768系列中，分辨率为 `768`。
+
+1. 批量大小
+
+    指定同时学习多少个数据。这取决于GPU的VRAM大小和学习分辨率。详细信息将在后面说明。此外，fine tuning/DreamBooth/LoRA等也会影响批量大小，请查看各个脚本的说明。
+
+1. 文件夹指定
+
+    指定用于学习的图像和正则化图像（仅在使用时）的文件夹。指定包含图像数据的文件夹。
+
+1. identifier 和 class 的指定
+
+    如前所述，与示例相同。
+
+1. 迭代次数
+
+    将在后面说明。
+
+### 关于重复次数
+
+重复次数用于调整正则化图像和训练用图像的数量。由于正则化图像的数量多于训练用图像，因此需要重复使用训练用图像来达到一对一的比例，从而实现训练。
+
+请将重复次数指定为“ __训练用图像的重复次数×训练用图像的数量≥正则化图像的重复次数×正则化图像的数量__ ”。
+
+（1个epoch（数据一周一次）的数据量为“训练用图像的重复次数×训练用图像的数量”。如果正则化图像的数量多于这个值，则剩余的正则化图像将不会被使用。）
+
+## 步骤 3. 学习
+
+请根据每个文档的参考进行学习。
+
+# DreamBooth，标题方式（可使用规范化图像）
+
+在此方式中，每个图像都将通过标题进行学习。
+
+## 步骤 1. 准备标题文件
+
+请将与图像具有相同文件名且扩展名为 `.caption`（可以在设置中更改）的文件放置在用于训练图像的文件夹中。每个文件应该只有一行。编码为 `UTF-8`。
+
+## 步骤 2. 决定是否使用规范化图像，并在使用时生成规范化图像
+
+与class+identifier格式相同。可以在规范化图像上附加标题，但通常不需要。
+
+## 步骤 2. 编写设置文件
+
+创建一个文本文件并将扩展名更改为 `.toml`。例如，可以按以下方式进行记录。
+
+```toml
+[general]
+enable_bucket = true                        # Aspect Ratio Bucketingを使うか否か
+
+[[datasets]]
+resolution = 512                            # 学習解像度
+batch_size = 4                              # 批量大小
+
+  [[datasets.subsets]]
+  image_dir = 'C:\hoge'                     # 指定包含训练图像的文件夹
+  caption_extension = '.caption'            # 使用字幕文件扩展名 .txt 时重写
+  num_repeats = 10                          # 训练图像的迭代次数
+
+  # 以下仅在使用正则化图像时进行描述。不使用则删除
+  [[datasets.subsets]]
+  is_reg = true
+  image_dir = 'C:\reg'                      #指定包含正则化图像的文件夹
+  class_tokens = 'girl'                     # class を指定
+  num_repeats = 1                           # 
+正则化图像的迭代次数，基本上1就可以了
+```
+
+基本上，您可以通过仅重写以下位置来学习。除非另有说明，否则与类+标识符方法相同。
+
+1. 学习分辨率
+2. 批量大小
+3. 文件夹指定
+4. 标题文件的扩展名
+
+    可以指定任意的扩展名。
+5. 重复次数
+
+## 步骤 3. 学习
+
+请参考每个文档进行学习。
+
+# 微调方法
+
+## 步骤 1. 准备元数据
+
+将标题和标签整合到管理文件中称为元数据。它的扩展名为 `.json`，格式为json。由于创建方法较长，因此在本文档的末尾进行了描述。
+
+## 步骤 2. 编写设置文件
+
+创建一个文本文件，将扩展名设置为 `.toml`。例如，可以按以下方式编写：
+```toml
+[general]
+shuffle_caption = true
+keep_tokens = 1
+
+[[datasets]]
+resolution = 512                                    # 图像分辨率
+batch_size = 4                                      # 批量大小
+
+  [[datasets.subsets]]
+  image_dir = 'C:\piyo'                             # 指定包含训练图像的文件夹
+  metadata_file = 'C:\piyo\piyo_md.json'            # 元数据文件名
+```
+
+基本上，您可以通过仅重写以下位置来学习。如无特别说明，与DreamBooth相同，类+标识符方式。
+
+1. 学习解像度
+2. 批次大小
+3. 指定文件夹
+4. 元数据文件名
+
+    指定使用后面所述方法创建的元数据文件。
+
+
+## 第三步：学习
+
+请参考各个文档进行学习。
+
+# 学习中使用的术语简单解释
+
+由于省略了细节并且我自己也没有完全理解，因此请自行查阅详细信息。
+
+## 微调（fine tuning）
+
+指训练模型并微调其性能。具体含义因用法而异，但在 Stable Diffusion 中，狭义的微调是指使用图像和标题进行训练模型。DreamBooth 可视为狭义微调的一种特殊方法。广义的微调包括 LoRA、Textual Inversion、Hypernetworks 等，包括训练模型的所有内容。
+
+## 步骤（step）
+
+粗略地说，每次在训练数据上进行一次计算即为一步。具体来说，“将训练数据的标题传递给当前模型，将生成的图像与训练数据的图像进行比较，稍微更改模型，以使其更接近训练数据”即为一步。
+
+## 批次大小（batch size）
+
+批次大小指定每个步骤要计算多少数据。批量计算可以提高速度。一般来说，批次大小越大，精度也越高。
+
+“批次大小×步数”是用于训练的数据数量。因此，建议减少步数以增加批次大小。
+
+（但是，例如，“批次大小为 1，步数为 1600”和“批次大小为 4，步数为 400”将不会产生相同的结果。如果使用相同的学习速率，通常后者会导致模型欠拟合。请尝试增加学习率（例如 `2e-6`），将步数设置为 500 等。）
+
+批次大小越大，GPU 内存消耗就越大。如果内存不足，将导致错误，或者在边缘时将导致训练速度降低。建议在任务管理器或 `nvidia-smi` 命令中检查使用的内存量进行调整。
+
+另外，批次是指“一块数据”的意思。
+
+## 学习率
+
+ 学习率指的是每个步骤中改变的程度。如果指定一个大的值，学习速度就会加快，但是可能会出现变化太大导致模型崩溃或无法达到最佳状态的情况。如果指定一个小的值，学习速度会变慢，也可能无法达到最佳状态。
+
+在fine tuning、DreamBooth、LoRA等过程中，学习率会有很大的差异，并且也会受到训练数据、所需训练的模型、批量大小和步骤数等因素的影响。建议从一般的值开始，观察训练状态并逐渐调整。
+
+默认情况下，整个训练过程中学习率是固定的。但是可以通过调度程序指定学习率如何变化，因此结果也会有所不同。
+
+## 时代（epoch）
+
+Epoch指的是训练数据被完整训练一遍（即数据一周）的情况。如果指定了重复次数，则在重复后的数据一周后，就是1个epoch。
+
+1个epoch的步骤数通常为“数据量÷批量大小”，但如果使用Aspect Ratio Bucketing，则略微增加（由于不同bucket的数据不能在同一个批次中，因此步骤数会增加）。
+
+## 纵横比分桶（Aspect Ratio Bucketing)
+
+Stable Diffusion 的 v1 是以 512\*512 的分辨率进行训练的，但同时也可以在其他分辨率下进行训练，例如 256\*1024 和 384\*640。这样可以减少裁剪的部分，期望更准确地学习图像和标题之间的关系。
+
+此外，由于可以在任意分辨率下进行训练，因此不再需要事先统一图像数据的纵横比。
+
+该设置在配置中有效，可以切换，但在此之前的配置文件示例中已启用（设置为 `true`）。
+
+学习分辨率将根据参数所提供的分辨率面积（即内存使用量）进行调整，以64像素为单位（默认值，可更改）在纵横方向上进行调整和创建。
+
+在机器学习中，通常需要将所有输入大小统一，但实际上只要在同一批次中统一即可。 NovelAI 所说的分桶(bucketing) 指的是，预先将训练数据按照纵横比分类到每个学习分辨率下，并通过使用每个 bucket 内的图像创建批次来统一批次图像大小。
+
+# 以前的指定格式（不使用 .toml 文件，而是使用命令行选项指定）
+
+这是一种通过命令行选项而不是指定 .toml 文件的方法。有 DreamBooth 类+标识符方法、DreamBooth 标题方法、微调方法三种方式。
+
+## DreamBooth、类+标识符方式
+
+指定文件夹名称以指定迭代次数。还要使用 `train_data_dir` 和 `reg_data_dir` 选项。
+
+### 第1步。准备用于训练的图像
+
+创建一个用于存储训练图像的文件夹。__此外__，按以下名称创建目录。
+
+```
+<迭代次数>_<标识符> <类别>
+```
+
+不要忘记下划线``_``。
+
+例如，如果在名为“sls frog”的提示下重复数据 20 次，则为“20_sls frog”。如下所示：
+
+![image](https://user-images.githubusercontent.com/52813779/210770636-1c851377-5936-4c15-90b7-8ac8ad6c2074.png)
+
+### 多个类别、多个标识符的学习
+
+该方法很简单，在用于训练的图像文件夹中，需要准备多个文件夹，每个文件夹都是以“重复次数_<标识符> <类别>”命名的，同样，在正则化图像文件夹中，也需要准备多个文件夹，每个文件夹都是以“重复次数_<类别>”命名的。
+
+例如，如果要同时训练“sls青蛙”和“cpc兔子”，则应按以下方式准备文件夹。
+
+![image](https://user-images.githubusercontent.com/52813779/210777933-a22229db-b219-4cd8-83ca-e87320fc4192.png)
+
+如果一个类别包含多个对象，可以只使用一个正则化图像文件夹。例如，如果在1girl类别中有角色A和角色B，则可以按照以下方式处理：
+
+- train_girls
+  - 10_sls 1girl
+  - 10_cpc 1girl
+- reg_girls
+  - 1_1girl
+
+### step 2. 准备正规化图像
+
+这是使用规则化图像时的过程。
+
+创建一个文件夹来存储规则化的图像。 __此外，__ 创建一个名为``<repeat count>_<class>`` 的目录。
+
+例如，使用提示“frog”并且不重复数据（仅一次）：
+![image](https://user-images.githubusercontent.com/52813779/210770897-329758e5-3675-49f1-b345-c135f1725832.png)
+
+
+步骤3. 执行学习
+
+执行每个学习脚本。使用 `--train_data_dir` 选项指定包含训练数据文件夹的父文件夹（不是包含图像的文件夹），使用 `--reg_data_dir` 选项指定包含正则化图像的父文件夹（不是包含图像的文件夹）。
+
+## DreamBooth，带标题方式
+
+在包含训练图像和正则化图像的文件夹中，将与图像具有相同文件名的文件.caption（可以使用选项进行更改）放置在该文件夹中，然后从该文件中加载标题作为提示进行学习。
+
+※文件夹名称（标识符类）不再用于这些图像的训练。
+
+默认的标题文件扩展名为.caption。可以使用学习脚本的 `--caption_extension` 选项进行更改。 使用 `--shuffle_caption` 选项，同时对每个逗号分隔的部分进行学习时会对学习时的标题进行混洗。
+
+## 微调方式
+
+创建元数据的方式与使用配置文件相同。 使用 `in_json` 选项指定元数据文件。
+
+# 学习过程中的样本输出
+
+通过在训练中使用模型生成图像，可以检查学习进度。将以下选项指定为学习脚本。
+
+- `--sample_every_n_steps` / `--sample_every_n_epochs`
+    
+    指定要采样的步数或纪元数。为这些数字中的每一个输出样本。如果两者都指定，则 epoch 数优先。
+- `--sample_prompts`
+
+    指定示例输出的提示文件。
+
+- `--sample_sampler`
+
+    指定用于采样输出的采样器。
+    `'ddim', 'pndm', 'heun', 'dpmsolver', 'dpmsolver++', 'dpmsingle', 'k_lms', 'k_euler', 'k_euler_a', 'k_dpm_2', 'k_dpm_2_a'`が選べます。
+
+要输出样本，您需要提前准备一个包含提示的文本文件。每行输入一个提示。
+
+```txt
+# prompt 1
+masterpiece, best quality, 1girl, in white shirts, upper body, looking at viewer, simple background --n low quality, worst quality, bad anatomy,bad composition, poor, low effort --w 768 --h 768 --d 1 --l 7.5 --s 28
+
+# prompt 2
+masterpiece, best quality, 1boy, in business suit, standing at street, looking back --n low quality, worst quality, bad anatomy,bad composition, poor, low effort --w 576 --h 832 --d 2 --l 5.5 --s 40
+```
+
+以“#”开头的行是注释。您可以使用“`--` + 小写字母”为生成的图像指定选项，例如 `--n`。您可以使用：
+
+- `--n` 否定提示到下一个选项。
+- `--w` 指定生成图像的宽度。
+- `--h` 指定生成图像的高度。
+- `--d` 指定生成图像的种子。
+- `--l` 指定生成图像的 CFG 比例。
+- `--s` 指定生成过程中的步骤数。
+
+
+# 每个脚本通用的常用选项
+
+文档更新可能跟不上脚本更新。在这种情况下，请使用 `--help` 选项检查可用选项。
+## 学习模型规范
+
+- `--v2` / `--v_parameterization`
+    
+   如果使用 Hugging Face 的 stable-diffusion-2-base 或来自它的微调模型作为学习目标模型（对于在推理时指示使用 `v2-inference.yaml` 的模型），`- 当使用-v2` 选项与 stable-diffusion-2、768-v-ema.ckpt 及其微调模型（对于在推理过程中使用 `v2-inference-v.yaml` 的模型），`- 指定两个 -v2`和 `--v_parameterization` 选项。
+
+    以下几点在 Stable Diffusion 2.0 中发生了显着变化。
+
+    1.  使用分词器
+    2. 使用哪个Text Encoder，使用哪个输出层（2.0使用倒数第二层）
+    3. Text Encoder的输出维度(768->1024)
+    4. U-Net的结构（CrossAttention的头数等）
+    5. v-parameterization（采样方式好像变了）
+
+    其中碱基使用1-4个，非碱基使用1-5个（768-v）。使用 1-4 进行 v2 选择，使用 5 进行 v_parameterization 选择。
+-`--pretrained_model_name_or_path`
+    
+    指定要从中执行额外训练的模型。您可以指定稳定扩散检查点文件（.ckpt 或 .safetensors）、扩散器本地磁盘上的模型目录或扩散器模型 ID（例如“stabilityai/stable-diffusion-2”）。
+## 学习设置
+
+- `--output_dir` 
+
+    指定训练后保存模型的文件夹。
+    
+- `--output_name` 
+    
+    指定不带扩展名的模型文件名。
+    
+- `--dataset_config` 
+
+    指定描述数据集配置的 .toml 文件。
+
+- `--max_train_steps` / `--max_train_epochs`
+
+    指定要学习的步数或纪元数。如果两者都指定，则 epoch 数优先。
+- 
+- `--mixed_precision`
+
+ 训练混合精度以节省内存。指定像`--mixed_precision = "fp16"`。与无混合精度（默认）相比，精度可能较低，但训练所需的 GPU 内存明显较少。
+    
+    （在RTX30系列以后也可以指定`bf16`，请配合您在搭建环境时做的加速设置）。    
+- `--gradient_checkpointing`
+
+  通过逐步计算权重而不是在训练期间一次计算所有权重来减少训练所需的 GPU 内存量。关闭它不会影响准确性，但打开它允许更大的批量大小，所以那里有影响。
+    
+    另外，打开它通常会减慢速度，但可以增加批量大小，因此总的学习时间实际上可能会更快。
+
+- `--xformers` / `--mem_eff_attn`
+
+   当指定 xformers 选项时，使用 xformers 的 CrossAttention。如果未安装 xformers 或发生错误（取决于环境，例如 `mixed_precision="no"`），请指定 `mem_eff_attn` 选项而不是使用 CrossAttention 的内存节省版本（xformers 比 慢）。
+- `--save_precision`
+
+   指定保存时的数据精度。为 save_precision 选项指定 float、fp16 或 bf16 将以该格式保存模型（在 DreamBooth 中保存 Diffusers 格式时无效，微调）。当您想缩小模型的尺寸时请使用它。
+- `--save_every_n_epochs` / `--save_state` / `--resume`
+    为 save_every_n_epochs 选项指定一个数字可以在每个时期的训练期间保存模型。
+
+    如果同时指定save_state选项，学习状态包括优化器的状态等都会一起保存。。保存目的地将是一个文件夹。
+    
+    学习状态输出到目标文件夹中名为“<output_name>-??????-state”（??????是纪元数）的文件夹中。长时间学习时请使用。
+
+    使用 resume 选项从保存的训练状态恢复训练。指定学习状态文件夹（其中的状态文件夹，而不是 `output_dir`）。
+
+    请注意，由于 Accelerator 规范，epoch 数和全局步数不会保存，即使恢复时它们也从 1 开始。
+- `--save_model_as` （DreamBooth, fine tuning 仅有的）
+
+  您可以从 `ckpt, safetensors, diffusers, diffusers_safetensors` 中选择模型保存格式。
+ 
+- `--save_model_as=safetensors` 指定喜欢当读取稳定扩散格式（ckpt 或安全张量）并以扩散器格式保存时，缺少的信息通过从 Hugging Face 中删除 v1.5 或 v2.1 信息来补充。
+    
+- `--clip_skip`
+    
+    `2`  如果指定，则使用文本编码器 (CLIP) 的倒数第二层的输出。如果省略 1 或选项，则使用最后一层。
+
+    *SD2.0默认使用倒数第二层，学习SD2.0时请不要指定。
+
+    如果被训练的模型最初被训练为使用第二层，则 2 是一个很好的值。
+
+    如果您使用的是最后一层，那么整个模型都会根据该假设进行训练。因此，如果再次使用第二层进行训练，可能需要一定数量的teacher数据和更长时间的学习才能得到想要的学习结果。
+- `--max_token_length`
+
+    默认值为 75。您可以通过指定“150”或“225”来扩展令牌长度来学习。使用长字幕学习时指定。
+    
+    但由于学习时token展开的规范与Automatic1111的web UI（除法等规范）略有不同，如非必要建议用75学习。
+
+    与clip_skip一样，学习与模型学习状态不同的长度可能需要一定量的teacher数据和更长的学习时间。
+
+- `--persistent_data_loader_workers`
+
+    在 Windows 环境中指定它可以显着减少时期之间的延迟。
+
+- `--max_data_loader_n_workers`
+
+    指定数据加载的进程数。大量的进程会更快地加载数据并更有效地使用 GPU，但会消耗更多的主内存。默认是"`8`或者`CPU并发执行线程数 - 1`，取小者"，所以如果主存没有空间或者GPU使用率大概在90%以上，就看那些数字和 `2` 或将其降低到大约 `1`。
+- `--logging_dir` / `--log_prefix`
+
+   保存学习日志的选项。在 logging_dir 选项中指定日志保存目标文件夹。以 TensorBoard 格式保存日志。
+
+    例如，如果您指定 --logging_dir=logs，将在您的工作文件夹中创建一个日志文件夹，并将日志保存在日期/时间文件夹中。
+    此外，如果您指定 --log_prefix 选项，则指定的字符串将添加到日期和时间之前。使用“--logging_dir=logs --log_prefix=db_style1_”进行识别。
+
+    要检查 TensorBoard 中的日志，请打开另一个命令提示符并在您的工作文件夹中键入：
+    ```
+    tensorboard --logdir=logs
+    ```
+
+   我觉得tensorboard会在环境搭建的时候安装，如果没有安装，请用`pip install tensorboard`安装。）
+
+    然后打开浏览器到http://localhost:6006/就可以看到了。
+- `--noise_offset`
+本文的实现：https://www.crosslabs.org//blog/diffusion-with-offset-noise
+    
+    看起来它可能会为整体更暗和更亮的图像产生更好的结果。它似乎对 LoRA 学习也有效。指定一个大约 0.1 的值似乎很好。
+
+- `--debug_dataset`
+
+   通过添加此选项，您可以在学习之前检查将学习什么样的图像数据和标题。按 Esc 退出并返回命令行。按 `S` 进入下一步（批次），按 `E` 进入下一个纪元。
+
+    *图片在 Linux 环境（包括 Colab）下不显示。
+
+- `--vae`
+
+   如果您在 vae 选项中指定稳定扩散检查点、VAE 检查点文件、扩散模型或 VAE（两者都可以指定本地或拥抱面模型 ID），则该 VAE 用于学习（缓存时的潜伏）或在学习过程中获得潜伏）。
+
+    对于 DreamBooth 和微调，保存的模型将包含此 VAE
+
+- `--cache_latents`
+
+  在主内存中缓存 VAE 输出以减少 VRAM 使用。除 flip_aug 之外的任何增强都将不可用。此外，整体学习速度略快。
+- `--min_snr_gamma`
+
+    指定最小 SNR 加权策略。细节是[这里](https://github.com/kohya-ss/sd-scripts/pull/308)请参阅。论文中推荐`5`。
+
+## 优化器相关
+
+- `--optimizer_type`
+    -- 指定优化器类型。您可以指定
+    - AdamW : [torch.optim.AdamW](https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html)
+    - 与过去版本中未指定选项时相同
+    - AdamW8bit : 同上
+    - 与过去版本中指定的 --use_8bit_adam 相同
+    - Lion : https://github.com/lucidrains/lion-pytorch
+    - 与过去版本中指定的 --use_lion_optimizer 相同
+    - SGDNesterov : [torch.optim.SGD](https://pytorch.org/docs/stable/generated/torch.optim.SGD.html), nesterov=True
+    - SGDNesterov8bit : 参数同上
+    - DAdaptation(DAdaptAdamPreprint) : https://github.com/facebookresearch/dadaptation
+    - DAdaptAdam : 参数同上
+    - DAdaptAdaGrad : 参数同上
+    - DAdaptAdan : 参数同上
+    - DAdaptAdanIP : 参数同上
+    - DAdaptLion : 参数同上
+    - DAdaptSGD : 参数同上
+    - Prodigy : https://github.com/konstmish/prodigy
+    - AdaFactor : [Transformers AdaFactor](https://huggingface.co/docs/transformers/main_classes/optimizer_schedules)
+    - 任何优化器
+
+- `--learning_rate`
+
+   指定学习率。合适的学习率取决于学习脚本，所以请参考每个解释。
+- `--lr_scheduler` / `--lr_warmup_steps` / `--lr_scheduler_num_cycles` / `--lr_scheduler_power`
+  
+    学习率的调度程序相关规范。
+
+    使用 lr_scheduler 选项，您可以从线性、余弦、cosine_with_restarts、多项式、常数、constant_with_warmup 或任何调度程序中选择学习率调度程序。默认值是常量。
+    
+    使用 lr_warmup_steps，您可以指定预热调度程序的步数（逐渐改变学习率）。
+    
+    lr_scheduler_num_cycles 是 cosine with restarts 调度器中的重启次数，lr_scheduler_power 是多项式调度器中的多项式幂。
+
+    有关详细信息，请自行研究。
+
+    要使用任何调度程序，请像使用任何优化器一样使用“--scheduler_args”指定可选参数。
+### 关于指定优化器
+
+使用 --optimizer_args 选项指定优化器选项参数。可以以key=value的格式指定多个值。此外，您可以指定多个值，以逗号分隔。例如，要指定 AdamW 优化器的参数，``--optimizer_args weight_decay=0.01 betas=.9,.999``。
+
+指定可选参数时，请检查每个优化器的规格。
+一些优化器有一个必需的参数，如果省略它会自动添加（例如 SGDNesterov 的动量）。检查控制台输出。
+
+D-Adaptation 优化器自动调整学习率。学习率选项指定的值不是学习率本身，而是D-Adaptation决定的学习率的应用率，所以通常指定1.0。如果您希望 Text Encoder 的学习率是 U-Net 的一半，请指定 ``--text_encoder_lr=0.5 --unet_lr=1.0``。
+如果指定 relative_step=True，AdaFactor 优化器可以自动调整学习率（如果省略，将默认添加）。自动调整时，学习率调度器被迫使用 adafactor_scheduler。此外，指定 scale_parameter 和 warmup_init 似乎也不错。
+
+自动调整的选项类似于``--optimizer_args "relative_step=True" "scale_parameter=True" "warmup_init=True"``。
+
+如果您不想自动调整学习率，请添加可选参数 ``relative_step=False``。在那种情况下，似乎建议将 constant_with_warmup 用于学习率调度程序，而不要为梯度剪裁范数。所以参数就像``--optimizer_type=adafactor --optimizer_args "relative_step=False" --lr_scheduler="constant_with_warmup" --max_grad_norm=0.0``。
+
+### 使用任何优化器
+
+使用 ``torch.optim`` 优化器时，仅指定类名（例如 ``--optimizer_type=RMSprop``），使用其他模块的优化器时，指定“模块名.类名”。（例如``--optimizer_type=bitsandbytes.optim.lamb.LAMB``）。
+
+（内部仅通过 importlib 未确认操作。如果需要，请安装包。）
+<!-- 
+## 使用任意大小的图像进行训练 --resolution
+你可以在广场外学习。请在分辨率中指定“宽度、高度”，如“448,640”。宽度和高度必须能被 64 整除。匹配训练图像和正则化图像的大小。
+
+就我个人而言，我经常生成垂直长的图像，所以我有时会用“448、640”来学习。
+
+## 纵横比分桶 --enable_bucket / --min_bucket_reso / --max_bucket_reso
+它通过指定 enable_bucket 选项来启用。 Stable Diffusion 在 512x512 分辨率下训练，但也在 256x768 和 384x640 等分辨率下训练。
+
+如果指定此选项，则不需要将训练图像和正则化图像统一为特定分辨率。从多种分辨率（纵横比）中进行选择，并在该分辨率下学习。
+由于分辨率为 64 像素，纵横比可能与原始图像不完全相同。
+
+您可以使用 min_bucket_reso 选项指定分辨率的最小大小，使用 max_bucket_reso 指定最大大小。默认值分别为 256 和 1024。
+例如，将最小尺寸指定为 384 将不会使用 256x1024 或 320x768 等分辨率。
+如果将分辨率增加到 768x768，您可能需要将 1280 指定为最大尺寸。
+
+启用 Aspect Ratio Ratio Bucketing 时，最好准备具有与训练图像相似的各种分辨率的正则化图像。
+
+（因为一批中的图像不偏向于训练图像和正则化图像。
+
+## 扩充 --color_aug / --flip_aug
+增强是一种通过在学习过程中动态改变数据来提高模型性能的方法。在使用 color_aug 巧妙地改变色调并使用 flip_aug 左右翻转的同时学习。
+
+由于数据是动态变化的，因此不能与 cache_latents 选项一起指定。
+
+## 使用 fp16 梯度训练（实验特征）--full_fp16
+如果指定 full_fp16 选项，梯度从普通 float32 变为 float16 (fp16) 并学习（它似乎是 full fp16 学习而不是混合精度）。
+结果，似乎 SD1.x 512x512 大小可以在 VRAM 使用量小于 8GB 的​​情况下学习，而 SD2.x 512x512 大小可以在 VRAM 使用量小于 12GB 的情况下学习。
+
+预先在加速配置中指定 fp16，并可选择设置 ``mixed_precision="fp16"``（bf16 不起作用）。
+
+为了最大限度地减少内存使用，请使用 xformers、use_8bit_adam、cache_latents、gradient_checkpointing 选项并将 train_batch_size 设置为 1。
+
+（如果你负担得起，逐步增加 train_batch_size 应该会提高一点精度。）
+
+它是通过修补 PyTorch 源代码实现的（已通过 PyTorch 1.12.1 和 1.13.0 确认）。准确率会大幅下降，途中学习失败的概率也会增加。
+学习率和步数的设置似乎很严格。请注意它们并自行承担使用它们的风险。
+-->
+
+# 创建元数据文件
+
+## 准备教师资料
+
+如上所述准备好你要学习的图像数据，放在任意文件夹中。
+
+例如，存储这样的图像：
+
+![教师数据文件夹的屏幕截图](https://user-images.githubusercontent.com/52813779/208907739-8e89d5fa-6ca8-4b60-8927-f484d2a9ae04.png)
+
+## 自动字幕
+
+如果您只想学习没有标题的标签，请跳过。
+
+另外，手动准备字幕时，请准备在与教师数据图像相同的目录下，文件名相同，扩展名.caption等。每个文件应该是只有一行的文本文件。
+### 使用 BLIP 添加字幕
+
+最新版本不再需要 BLIP 下载、权重下载和额外的虚拟环境。按原样工作。
+
+运行 finetune 文件夹中的 make_captions.py。
+
+```
+python finetune\make_captions.py --batch_size <バッチサイズ> <教師データフォルダ>
+```
+
+如果batch size为8，训练数据放在父文件夹train_data中，则会如下所示
+```
+python finetune\make_captions.py --batch_size 8 ..\train_data
+```
+
+字幕文件创建在与教师数据图像相同的目录中，具有相同的文件名和扩展名.caption。
+
+根据 GPU 的 VRAM 容量增加或减少 batch_size。越大越快（我认为 12GB 的 VRAM 可以多一点）。
+您可以使用 max_length 选项指定标题的最大长度。默认值为 75。如果使用 225 的令牌长度训练模型，它可能会更长。
+您可以使用 caption_extension 选项更改标题扩展名。默认为 .caption（.txt 与稍后描述的 DeepDanbooru 冲突）。
+如果有多个教师数据文件夹，则对每个文件夹执行。
+
+请注意，推理是随机的，因此每次运行时结果都会发生变化。如果要修复它，请使用 --seed 选项指定一个随机数种子，例如 `--seed 42`。
+
+其他的选项，请参考help with `--help`（好像没有文档说明参数的含义，得看源码）。
+
+默认情况下，会生成扩展名为 .caption 的字幕文件。
+
+![caption生成的文件夹](https://user-images.githubusercontent.com/52813779/208908845-48a9d36c-f6ee-4dae-af71-9ab462d1459e.png)
+
+例如，标题如下：
+
+![字幕和图像](https://user-images.githubusercontent.com/52813779/208908947-af936957-5d73-4339-b6c8-945a52857373.png)
+
+## 由 DeepDanbooru 标记
+
+如果不想给danbooru标签本身打标签，请继续“标题和标签信息的预处理”。
+
+标记是使用 DeepDanbooru 或 WD14Tagger 完成的。 WD14Tagger 似乎更准确。如果您想使用 WD14Tagger 进行标记，请跳至下一章。
+### 环境布置
+
+将 DeepDanbooru https://github.com/KichangKim/DeepDanbooru 克隆到您的工作文件夹中，或下载并展开 zip。我解压缩了它。
+另外，从 DeepDanbooru 发布页面 https://github.com/KichangKim/DeepDanbooru/releases 上的“DeepDanbooru 预训练模型 v3-20211112-sgd-e28”的资产下载 deepdanbooru-v3-20211112-sgd-e28.zip 并解压到 DeepDanbooru 文件夹。
+
+从下面下载。单击以打开资产并从那里下载。
+
+![DeepDanbooru下载页面](https://user-images.githubusercontent.com/52813779/208909417-10e597df-7085-41ee-bd06-3e856a1339df.png)
+
+做一个这样的目录结构
+
+![DeepDanbooru的目录结构](https://user-images.githubusercontent.com/52813779/208909486-38935d8b-8dc6-43f1-84d3-fef99bc471aa.png)
+为扩散器环境安装必要的库。进入 DeepDanbooru 文件夹并安装它（我认为它实际上只是添加了 tensorflow-io）。
+```
+pip install -r requirements.txt
+```
+
+接下来，安装 DeepDanbooru 本身。
+
+```
+pip install .
+```
+
+这样就完成了标注环境的准备工作。
+
+### 实施标记
+转到 DeepDanbooru 的文件夹并运行 deepdanbooru 进行标记。
+```
+deepdanbooru evaluate <教师资料夹> --project-path deepdanbooru-v3-20211112-sgd-e28 --allow-folder --save-txt
+```
+
+如果将训练数据放在父文件夹train_data中，则如下所示。
+```
+deepdanbooru evaluate ../train_data --project-path deepdanbooru-v3-20211112-sgd-e28 --allow-folder --save-txt
+```
+
+在与教师数据图像相同的目录中创建具有相同文件名和扩展名.txt 的标记文件。它很慢，因为它是一个接一个地处理的。
+
+如果有多个教师数据文件夹，则对每个文件夹执行。
+
+它生成如下。
+
+![DeepDanbooru生成的文件](https://user-images.githubusercontent.com/52813779/208909855-d21b9c98-f2d3-4283-8238-5b0e5aad6691.png)
+
+它会被这样标记（信息量很大...）。
+
+![DeepDanbooru标签和图片](https://user-images.githubusercontent.com/52813779/208909908-a7920174-266e-48d5-aaef-940aba709519.png)
+
+## WD14Tagger标记为
+
+此过程使用 WD14Tagger 而不是 DeepDanbooru。
+
+使用 Mr. Automatic1111 的 WebUI 中使用的标记器。我参考了这个 github 页面上的信息 (https://github.com/toriato/stable-diffusion-webui-wd14-tagger#mrsmilingwolfs-model-aka-waifu-diffusion-14-tagger)。
+
+初始环境维护所需的模块已经安装。权重自动从 Hugging Face 下载。
+### 实施标记
+
+运行脚本以进行标记。
+```
+python tag_images_by_wd14_tagger.py --batch_size <バッチサイズ> <教師データフォルダ>
+```
+
+如果将训练数据放在父文件夹train_data中，则如下所示
+```
+python tag_images_by_wd14_tagger.py --batch_size 4 ..\train_data
+```
+
+模型文件将在首次启动时自动下载到 wd14_tagger_model 文件夹（文件夹可以在选项中更改）。它将如下所示。
+![下载文件](https://user-images.githubusercontent.com/52813779/208910447-f7eb0582-90d6-49d3-a666-2b508c7d1842.png)
+
+在与教师数据图像相同的目录中创建具有相同文件名和扩展名.txt 的标记文件。
+![生成的标签文件](https://user-images.githubusercontent.com/52813779/208910534-ea514373-1185-4b7d-9ae3-61eb50bc294e.png)
+
+![标签和图片](https://user-images.githubusercontent.com/52813779/208910599-29070c15-7639-474f-b3e4-06bd5a3df29e.png)
+
+使用 thresh 选项，您可以指定确定的标签的置信度数以附加标签。默认值为 0.35，与 WD14Tagger 示例相同。较低的值给出更多的标签，但准确性较低。
+
+根据 GPU 的 VRAM 容量增加或减少 batch_size。越大越快（我认为 12GB 的 VRAM 可以多一点）。您可以使用 caption_extension 选项更改标记文件扩展名。默认为 .txt。
+
+您可以使用 model_dir 选项指定保存模型的文件夹。
+
+此外，如果指定 force_download 选项，即使有保存目标文件夹，也会重新下载模型。
+
+如果有多个教师数据文件夹，则对每个文件夹执行。
+
+## 预处理字幕和标签信息
+
+将字幕和标签作为元数据合并到一个文件中，以便从脚本中轻松处理。
+### 字幕预处理
+
+要将字幕放入元数据，请在您的工作文件夹中运行以下命令（如果您不使用字幕进行学习，则不需要运行它）（它实际上是一行，依此类推）。指定 `--full_path` 选项以将图像文件的完整路径存储在元数据中。如果省略此选项，则会记录相对路径，但 .toml 文件中需要单独的文件夹规范。
+```
+python merge_captions_to_metadata.py --full_path <教师资料夹>
+　  --in_json <要读取的元数据文件名> <元数据文件名>
+```
+
+元数据文件名是任意名称。
+如果训练数据为train_data，没有读取元数据文件，元数据文件为meta_cap.json，则会如下。
+```
+python merge_captions_to_metadata.py --full_path train_data meta_cap.json
+```
+
+您可以使用 caption_extension 选项指定标题扩展。
+
+如果有多个教师数据文件夹，请指定 full_path 参数并为每个文件夹执行。
+```
+python merge_captions_to_metadata.py --full_path 
+    train_data1 meta_cap1.json
+python merge_captions_to_metadata.py --full_path --in_json meta_cap1.json 
+    train_data2 meta_cap2.json
+```
+如果省略in_json，如果有写入目标元数据文件，将从那里读取并覆盖。
+
+__* 每次重写 in_json 选项和写入目标并写入单独的元数据文件是安全的。 __
+### 标签预处理
+
+同样，标签也收集在元数据中（如果标签不用于学习，则无需这样做）。
+```
+python merge_dd_tags_to_metadata.py --full_path <教师资料夹> 
+    --in_json <要读取的元数据文件名> <要写入的元数据文件名>
+```
+
+同样的目录结构，读取meta_cap.json和写入meta_cap_dd.json时，会是这样的。
+```
+python merge_dd_tags_to_metadata.py --full_path train_data --in_json meta_cap.json meta_cap_dd.json
+```
+
+如果有多个教师数据文件夹，请指定 full_path 参数并为每个文件夹执行。
+
+```
+python merge_dd_tags_to_metadata.py --full_path --in_json meta_cap2.json
+    train_data1 meta_cap_dd1.json
+python merge_dd_tags_to_metadata.py --full_path --in_json meta_cap_dd1.json 
+    train_data2 meta_cap_dd2.json
+```
+
+如果省略in_json，如果有写入目标元数据文件，将从那里读取并覆盖。
+__※ 通过每次重写 in_json 选项和写入目标，写入单独的元数据文件是安全的。 __
+### 标题和标签清理
+
+到目前为止，标题和DeepDanbooru标签已经被整理到元数据文件中。然而，自动标题生成的标题存在表达差异等微妙问题（※），而标签中可能包含下划线和评级（DeepDanbooru的情况下）。因此，最好使用编辑器的替换功能清理标题和标签。
+
+※例如，如果要学习动漫中的女孩，标题可能会包含girl/girls/woman/women等不同的表达方式。另外，将"anime girl"简单地替换为"girl"可能更合适。
+
+我们提供了用于清理的脚本，请根据情况编辑脚本并使用它。
+
+（不需要指定教师数据文件夹。将清理元数据中的所有数据。）
+
+```
+python clean_captions_and_tags.py <要读取的元数据文件名> <要写入的元数据文件名>
+```
+
+--in_json 请注意，不包括在内。例如：
+
+```
+python clean_captions_and_tags.py meta_cap_dd.json meta_clean.json
+```
+
+标题和标签的预处理现已完成。
+
+## 预先获取 latents
+
+※ 这一步骤并非必须。即使省略此步骤，也可以在训练过程中获取 latents。但是，如果在训练时执行 `random_crop` 或 `color_aug` 等操作，则无法预先获取 latents（因为每次图像都会改变）。如果不进行预先获取，则可以使用到目前为止的元数据进行训练。
+
+提前获取图像的潜在表达并保存到磁盘上。这样可以加速训练过程。同时进行 bucketing（根据宽高比对训练数据进行分类）。
+
+请在工作文件夹中输入以下内容。
+
+```
+python prepare_buckets_latents.py --full_path <教师资料夹>  
+    <要读取的元数据文件名> <要写入的元数据文件名> 
+    <要微调的模型名称或检查点> 
+    --batch_size <批量大小> 
+    --max_resolution <分辨率宽、高> 
+    --mixed_precision <准确性>
+```
+
+如果要从meta_clean.json中读取元数据，并将其写入meta_lat.json，使用模型model.ckpt，批处理大小为4，训练分辨率为512*512，精度为no（float32），则应如下所示。
+```
+python prepare_buckets_latents.py --full_path 
+    train_data meta_clean.json meta_lat.json model.ckpt 
+    --batch_size 4 --max_resolution 512,512 --mixed_precision no
+```
+
+教师数据文件夹中，latents以numpy的npz格式保存。
+
+您可以使用--min_bucket_reso选项指定最小分辨率大小，--max_bucket_reso指定最大大小。默认值分别为256和1024。例如，如果指定最小大小为384，则将不再使用分辨率为256 * 1024或320 * 768等。如果将分辨率增加到768 * 768等较大的值，则最好将最大大小指定为1280等。
+
+如果指定--flip_aug选项，则进行左右翻转的数据增强。虽然这可以使数据量伪造一倍，但如果数据不是左右对称的（例如角色外观、发型等），则可能会导致训练不成功。
+
+对于翻转的图像，也会获取latents，并保存名为\ *_flip.npz的文件，这是一个简单的实现。在fline_tune.py中不需要特定的选项。如果有带有\_flip的文件，则会随机加载带有和不带有flip的文件。
+
+即使VRAM为12GB，批量大小也可以稍微增加。分辨率以“宽度，高度”的形式指定，必须是64的倍数。分辨率直接影响fine tuning时的内存大小。在12GB VRAM中，512,512似乎是极限（*）。如果有16GB，则可以将其提高到512,704或512,768。即使分辨率为256,256等，VRAM 8GB也很难承受（因为参数、优化器等与分辨率无关，需要一定的内存）。
+
+*有报道称，在batch size为1的训练中，使用12GB VRAM和640,640的分辨率。 
+
+以下是bucketing结果的显示方式。
+
+![bucketing的結果](https://user-images.githubusercontent.com/52813779/208911419-71c00fbb-2ce6-49d5-89b5-b78d7715e441.png)
+
+如果有多个教师数据文件夹，请指定 full_path 参数并为每个文件夹执行
+
+```
+python prepare_buckets_latents.py --full_path  
+    train_data1 meta_clean.json meta_lat1.json model.ckpt 
+    --batch_size 4 --max_resolution 512,512 --mixed_precision no
+
+python prepare_buckets_latents.py --full_path 
+    train_data2 meta_lat1.json meta_lat2.json model.ckpt 
+    --batch_size 4 --max_resolution 512,512 --mixed_precision no
+
+```
+可以将读取源和写入目标设为相同，但分开设定更为安全。
+
+__※建议每次更改参数并将其写入另一个元数据文件，以确保安全性。__
diff --git a/docs/train_db_README-ja.md b/docs/train_db_README-ja.md
new file mode 100644
index 0000000000000000000000000000000000000000..0d0747bb41223a52a4f609f58eb1314639924913
--- /dev/null
+++ b/docs/train_db_README-ja.md
@@ -0,0 +1,167 @@
+DreamBoothのガイドです。
+
+[学習についての共通ドキュメント](./train_README-ja.md) もあわせてご覧ください。
+
+# 概要
+
+DreamBoothとは、画像生成モデルに特定の主題を追加学習し、それを特定の識別子で生成する技術です。[論文はこちら](https://arxiv.org/abs/2208.12242)。
+
+具体的には、Stable Diffusionのモデルにキャラや画風などを学ばせ、それを `shs` のような特定の単語で呼び出せる（生成画像に出現させる）ことができます。
+
+スクリプトは[DiffusersのDreamBooth](https://github.com/huggingface/diffusers/tree/main/examples/dreambooth)を元にしていますが、以下のような機能追加を行っています（いくつかの機能は元のスクリプト側もその後対応しています）。
+
+スクリプトの主な機能は以下の通りです。
+
+- 8bit Adam optimizerおよびlatentのキャッシュによる省メモリ化（[Shivam Shrirao氏版](https://github.com/ShivamShrirao/diffusers/tree/main/examples/dreambooth)と同様）。
+- xformersによる省メモリ化。
+- 512x512だけではなく任意サイズでの学習。
+- augmentationによる品質の向上。
+- DreamBoothだけではなくText Encoder+U-Netのfine tuningに対応。
+- Stable Diffusion形式でのモデルの読み書き。
+- Aspect Ratio Bucketing。
+- Stable Diffusion v2.0対応。
+
+# 学習の手順
+
+あらかじめこのリポジトリのREADMEを参照し、環境整備を行ってください。
+
+## データの準備
+
+[学習データの準備について](./train_README-ja.md) を参照してください。
+
+## 学習の実行
+
+スクリプトを実行します。最大限、メモリを節約したコマンドは以下のようになります（実際には1行で入力します）。それぞれの行を必要に応じて書き換えてください。12GB程度のVRAMで動作するようです。
+
+```
+accelerate launch --num_cpu_threads_per_process 1 train_db.py 
+    --pretrained_model_name_or_path=<.ckptまたは.safetensordまたはDiffusers版モデルのディレクトリ> 
+    --dataset_config=<データ準備で作成した.tomlファイル> 
+    --output_dir=<学習したモデルの出力先フォルダ>  
+    --output_name=<学習したモデル出力時のファイル名> 
+    --save_model_as=safetensors 
+    --prior_loss_weight=1.0 
+    --max_train_steps=1600 
+    --learning_rate=1e-6 
+    --optimizer_type="AdamW8bit" 
+    --xformers 
+    --mixed_precision="fp16" 
+    --cache_latents 
+    --gradient_checkpointing
+```
+
+`num_cpu_threads_per_process` には通常は1を指定するとよいようです。
+
+`pretrained_model_name_or_path` に追加学習を行う元となるモデルを指定します。Stable Diffusionのcheckpointファイル（.ckptまたは.safetensors）、Diffusersのローカルディスクにあるモデルディレクトリ、DiffusersのモデルID（"stabilityai/stable-diffusion-2"など）が指定できます。
+
+`output_dir` に学習後のモデルを保存するフォルダを指定します。`output_name` にモデルのファイル名を拡張子を除いて指定します。`save_model_as` でsafetensors形式での保存を指定しています。
+
+`dataset_config` に `.toml` ファイルを指定します。ファイル内でのバッチサイズ指定は、当初はメモリ消費を抑えるために `1` としてください。
+
+`prior_loss_weight` は正則化画像のlossの重みです。通常は1.0を指定します。
+
+学習させるステップ数 `max_train_steps` を1600とします。学習率 `learning_rate` はここでは1e-6を指定しています。
+
+省メモリ化のため `mixed_precision="fp16"` を指定します（RTX30 シリーズ以降では `bf16` も指定できます。環境整備時にaccelerateに行った設定と合わせてください）。また `gradient_checkpointing` を指定します。
+
+オプティマイザ（モデルを学習データにあうように最適化＝学習させるクラス）にメモリ消費の少ない 8bit AdamW を使うため、 `optimizer_type="AdamW8bit"` を指定します。
+
+`xformers` オプションを指定し、xformersのCrossAttentionを用います。xformersをインストールしていない場合やエラーとなる場合（環境にもよりますが `mixed_precision="no"` の場合など）、代わりに `mem_eff_attn` オプションを指定すると省メモリ版CrossAttentionを使用します（速度は遅くなります）。
+
+省メモリ化のため `cache_latents` オプションを指定してVAEの出力をキャッシュします。
+
+ある程度メモリがある場合は、`.toml` ファイルを編集してバッチサイズをたとえば `4` くらいに増やしてください（高速化と精度向上の可能性があります）。また `cache_latents` を外すことで augmentation が可能になります。
+
+### よく使われるオプションについて
+
+以下の場合には [学習の共通ドキュメント](./train_README-ja.md) の「よく使われるオプション」を参照してください。
+
+- Stable Diffusion 2.xまたはそこからの派生モデルを学習する
+- clip skipを2以上を前提としたモデルを学習する
+- 75トークンを超えたキャプションで学習する
+
+### DreamBoothでのステップ数について
+
+当スクリプトでは省メモリ化のため、ステップ当たりの学習回数が元のスクリプトの半分になっています（対象の画像と正則化画像を同一のバッチではなく別のバッチに分割して学習するため）。
+
+元のDiffusers版やXavierXiao氏のStable Diffusion版とほぼ同じ学習を行うには、ステップ数を倍にしてください。
+
+（学習画像と正則化画像をまとめてから shuffle するため厳密にはデータの順番が変わってしまいますが、学習には大きな影響はないと思います。）
+
+### DreamBoothでのバッチサイズについて
+
+モデル全体を学習するためLoRA等の学習に比べるとメモリ消費量は多くなります（fine tuningと同じ）。
+
+### 学習率について
+
+Diffusers版では5e-6ですがStable Diffusion版は1e-6ですので、上のサンプルでは1e-6を指定しています。
+
+### 以前の形式のデータセット指定をした場合のコマンドライン
+
+解像度やバッチサイズをオプションで指定します。コマンドラインの例は以下の通りです。
+
+```
+accelerate launch --num_cpu_threads_per_process 1 train_db.py 
+    --pretrained_model_name_or_path=<.ckptまたは.safetensordまたはDiffusers版モデルのディレクトリ> 
+    --train_data_dir=<学習用データのディレクトリ> 
+    --reg_data_dir=<正則化画像のディレクトリ> 
+    --output_dir=<学習したモデルの出力先ディレクトリ> 
+    --output_name=<学習したモデル出力時のファイル名> 
+    --prior_loss_weight=1.0 
+    --resolution=512 
+    --train_batch_size=1 
+    --learning_rate=1e-6 
+    --max_train_steps=1600 
+    --use_8bit_adam 
+    --xformers 
+    --mixed_precision="bf16" 
+    --cache_latents
+    --gradient_checkpointing
+```
+
+## 学習したモデルで画像生成する
+
+学習が終わると指定したフォルダに指定した名前でsafetensorsファイルが出力されます。
+
+v1.4/1.5およびその他の派生モデルの場合、このモデルでAutomatic1111氏のWebUIなどで推論できます。models\Stable-diffusionフォルダに置いてください。
+
+v2.xモデルでWebUIで画像生成する場合、モデルの仕様が記述された.yamlファイルが別途必要になります。v2.x baseの場合はv2-inference.yamlを、768/vの場合はv2-inference-v.yamlを、同じフォルダに置き、拡張子の前の部分をモデルと同じ名前にしてください。
+
+![image](https://user-images.githubusercontent.com/52813779/210776915-061d79c3-6582-42c2-8884-8b91d2f07313.png)
+
+各yamlファイルは[Stability AIのSD2.0のリポジトリ](https://github.com/Stability-AI/stablediffusion/tree/main/configs/stable-diffusion)にあります。
+
+# DreamBooth特有のその他の主なオプション
+
+すべてのオプションについては別文書を参照してください。
+
+## Text Encoderの学習を途中から行わない --stop_text_encoder_training
+
+stop_text_encoder_trainingオプションに数値を指定すると、そのステップ数以降はText Encoderの学習を行わずU-Netだけ学習します。場合によっては精度の向上が期待できるかもしれません。
+
+（恐らくText Encoderだけ先に過学習することがあり、それを防げるのではないかと推測していますが、詳細な影響は不明です。）
+
+## Tokenizerのパディングをしない --no_token_padding
+no_token_paddingオプションを指定するとTokenizerの出力をpaddingしません（Diffusers版の旧DreamBoothと同じ動きになります）。
+
+
+<!-- 
+bucketing（後述）を利用しかつaugmentation（後述）を使う場合の例は以下のようになります。
+
+```
+accelerate launch --num_cpu_threads_per_process 8 train_db.py 
+    --pretrained_model_name_or_path=<.ckptまたは.safetensordまたはDiffusers版モデルのディレクトリ> 
+    --train_data_dir=<学習用データのディレクトリ> 
+    --reg_data_dir=<正則化画像のディレクトリ> 
+    --output_dir=<学習したモデルの出力先ディレクトリ> 
+    --resolution=768,512 
+    --train_batch_size=20 --learning_rate=5e-6 --max_train_steps=800 
+    --use_8bit_adam --xformers --mixed_precision="bf16" 
+    --save_every_n_epochs=1 --save_state --save_precision="bf16" 
+    --logging_dir=logs 
+    --enable_bucket --min_bucket_reso=384 --max_bucket_reso=1280 
+    --color_aug --flip_aug --gradient_checkpointing --seed 42
+```
+
+
+-->
diff --git a/docs/train_db_README-zh.md b/docs/train_db_README-zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..d8ea5f3edae856248c334d7c2a779dc44a86c0f5
--- /dev/null
+++ b/docs/train_db_README-zh.md
@@ -0,0 +1,162 @@
+这是DreamBooth的指南。
+
+请同时查看[关于学习的通用文档](./train_README-zh.md)。
+
+# 概要
+
+DreamBooth是一种将特定主题添加到图像生成模型中进行学习，并使用特定识别子生成它的技术。论文链接。
+
+具体来说，它可以将角色和绘画风格等添加到Stable Diffusion模型中进行学习，并使用特定的单词（例如`shs`）来调用（呈现在生成的图像中）。
+
+脚本基于Diffusers的DreamBooth，但添加了以下功能（一些功能已在原始脚本中得到支持）。
+
+脚本的主要功能如下：
+
+- 使用8位Adam优化器和潜在变量的缓存来节省内存（与Shivam Shrirao版相似）。
+- 使用xformers来节省内存。
+- 不仅支持512x512，还支持任意尺寸的训练。
+- 通过数据增强来提高质量。
+- 支持DreamBooth和Text Encoder + U-Net的微调。
+- 支持以Stable Diffusion格式读写模型。
+- 支持Aspect Ratio Bucketing。
+- 支持Stable Diffusion v2.0。
+
+# 训练步骤
+
+请先参阅此存储库的README以进行环境设置。
+
+## 准备数据
+
+请参阅[有关准备训练数据的说明](./train_README-zh.md)。
+
+## 运行训练
+
+运行脚本。以下是最大程度地节省内存的命令（实际上，这将在一行中输入）。请根据需要修改每行。它似乎需要约12GB的VRAM才能运行。
+```
+accelerate launch --num_cpu_threads_per_process 1 train_db.py 
+    --pretrained_model_name_or_path=<.ckpt或.safetensord或Diffusers版模型的目录>
+    --dataset_config=<数据准备时创建的.toml文件>
+    --output_dir=<训练模型的输出目录>
+    --output_name=<训练模型输出时的文件名>
+    --save_model_as=safetensors 
+    --prior_loss_weight=1.0 
+    --max_train_steps=1600 
+    --learning_rate=1e-6 
+    --optimizer_type="AdamW8bit" 
+    --xformers 
+    --mixed_precision="fp16" 
+    --cache_latents 
+    --gradient_checkpointing
+```
+`num_cpu_threads_per_process` 通常应该设置为1。
+
+`pretrained_model_name_or_path` 指定要进行追加训练的基础模型。可以指定 Stable Diffusion 的 checkpoint 文件（.ckpt 或 .safetensors）、Diffusers 的本地模型目录或模型 ID（如 "stabilityai/stable-diffusion-2"）。
+
+`output_dir` 指定保存训练后模型的文件夹。在 `output_name` 中指定模型文件名，不包括扩展名。使用 `save_model_as` 指定以 safetensors 格式保存。
+
+在 `dataset_config` 中指定 `.toml` 文件。初始批处理大小应为 `1`，以减少内存消耗。
+
+`prior_loss_weight` 是正则化图像损失的权重。通常设为1.0。
+
+将要训练的步数 `max_train_steps` 设置为1600。在这里，学习率 `learning_rate` 被设置为1e-6。
+
+为了节省内存，设置 `mixed_precision="fp16"`（在 RTX30 系列及更高版本中也可以设置为 `bf16`）。同时指定 `gradient_checkpointing`。
+
+为了使用内存消耗较少的 8bit AdamW 优化器（将模型优化为适合于训练数据的状态），指定 `optimizer_type="AdamW8bit"`。
+
+指定 `xformers` 选项，并使用 xformers 的 CrossAttention。如果未安装 xformers 或出现错误（具体情况取决于环境，例如使用 `mixed_precision="no"`），则可以指定 `mem_eff_attn` 选项以使用省内存版的 CrossAttention（速度会变慢）。
+
+为了节省内存，指定 `cache_latents` 选项以缓存 VAE 的输出。
+
+如果有足够的内存，请编辑 `.toml` 文件将批处理大小增加到大约 `4`（可能会提高速度和精度）。此外，取消 `cache_latents` 选项可以进行数据增强。
+
+### 常用选项
+
+对于以下情况，请参阅“常用选项”部分。
+
+- 学习 Stable Diffusion 2.x 或其衍生模型。
+- 学习基于 clip skip 大于等于2的模型。
+- 学习超过75个令牌的标题。
+
+### 关于DreamBooth中的步数
+
+为了实现省内存化，该脚本中每个步骤的学习次数减半（因为学习和正则化的图像在训练时被分为不同的批次）。
+
+要进行与原始Diffusers版或XavierXiao的Stable Diffusion版几乎相同的学习，请将步骤数加倍。
+
+（虽然在将学习图像和正则化图像整合后再打乱顺序，但我认为对学习没有太大影响。）
+
+关于DreamBooth的批量大小
+
+与像LoRA这样的学习相比，为了训练整个模型，内存消耗量会更大（与微调相同）。
+
+关于学习率
+
+在Diffusers版中，学习率为5e-6，而在Stable Diffusion版中为1e-6，因此在上面的示例中指定了1e-6。
+
+当使用旧格式的数据集指定命令行时
+
+使用选项指定分辨率和批量大小。命令行示例如下。
+```
+accelerate launch --num_cpu_threads_per_process 1 train_db.py 
+    --pretrained_model_name_or_path=<.ckpt或.safetensord或Diffusers版模型的目录> 
+    --train_data_dir=<训练数据的目录> 
+    --reg_data_dir=<正则化图像的目录> 
+    --output_dir=<训练后模型的输出目录> 
+    --output_name=<训练后模型输出文件的名称>  
+    --prior_loss_weight=1.0 
+    --resolution=512 
+    --train_batch_size=1 
+    --learning_rate=1e-6 
+    --max_train_steps=1600 
+    --use_8bit_adam 
+    --xformers 
+    --mixed_precision="bf16" 
+    --cache_latents
+    --gradient_checkpointing
+```
+
+## 使用训练好的模型生成图像
+
+训练完成后，将在指定的文件夹中以指定的名称输出safetensors文件。
+
+对于v1.4/1.5和其他派生模型，可以在此模型中使用Automatic1111先生的WebUI进行推断。请将其放置在models\Stable-diffusion文件夹中。
+
+对于使用v2.x模型在WebUI中生成图像的情况，需要单独的.yaml文件来描述模型的规格。对于v2.x base，需要v2-inference.yaml，对于768/v，则需要v2-inference-v.yaml。请将它们放置在相同的文件夹中，并将文件扩展名之前的部分命名为与模型相同的名称。
+![image](https://user-images.githubusercontent.com/52813779/210776915-061d79c3-6582-42c2-8884-8b91d2f07313.png)
+
+每个yaml文件都在[Stability AI的SD2.0存储库](https://github.com/Stability-AI/stablediffusion/tree/main/configs/stable-diffusion)……之中。
+
+# DreamBooth的其他主要选项
+
+有关所有选项的详细信息，请参阅另一份文档。
+
+## 不在中途开始对文本编码器进行训练 --stop_text_encoder_training
+
+如果在stop_text_encoder_training选项中指定一个数字，则在该步骤之后，将不再对文本编码器进行训练，只会对U-Net进行训练。在某些情况下，可能会期望提高精度。
+
+（我们推测可能会有时候仅仅文本编码器会过度学习，而这样做可以避免这种情况，但详细影响尚不清楚。）
+
+## 不进行分词器的填充 --no_token_padding
+
+如果指定no_token_padding选项，则不会对分词器的输出进行填充（与Diffusers版本的旧DreamBooth相同）。
+
+<!-- 
+如果使用分桶（bucketing）和数据增强（augmentation），则使用示例如下：
+```
+accelerate launch --num_cpu_threads_per_process 8 train_db.py 
+    --pretrained_model_name_or_path=<.ckpt或.safetensord或Diffusers版模型的目录> 
+    --train_data_dir=<训练数据的目录> 
+    --reg_data_dir=<正则化图像的目录> 
+    --output_dir=<训练后模型的输出目录>
+    --resolution=768,512 
+    --train_batch_size=20 --learning_rate=5e-6 --max_train_steps=800 
+    --use_8bit_adam --xformers --mixed_precision="bf16" 
+    --save_every_n_epochs=1 --save_state --save_precision="bf16" 
+    --logging_dir=logs 
+    --enable_bucket --min_bucket_reso=384 --max_bucket_reso=1280 
+    --color_aug --flip_aug --gradient_checkpointing --seed 42
+```
+
+
+-->
diff --git a/docs/train_network_README-ja.md b/docs/train_network_README-ja.md
new file mode 100644
index 0000000000000000000000000000000000000000..e620a8642742b0e5ead721d40a1c1a740603d5fe
--- /dev/null
+++ b/docs/train_network_README-ja.md
@@ -0,0 +1,481 @@
+# LoRAの学習について
+
+[LoRA: Low-Rank Adaptation of Large Language Models](https://arxiv.org/abs/2106.09685)（arxiv）、[LoRA](https://github.com/microsoft/LoRA)（github）をStable Diffusionに適用したものです。
+
+[cloneofsimo氏のリポジトリ](https://github.com/cloneofsimo/lora)を大いに参考にさせていただきました。ありがとうございます。
+
+通常のLoRAは Linear およぴカーネルサイズ 1x1 の Conv2d にのみ適用されますが、カーネルサイズ 3x3 のConv2dに適用を拡大することもできます。
+
+Conv2d 3x3への拡大は [cloneofsimo氏](https://github.com/cloneofsimo/lora) が最初にリリースし、KohakuBlueleaf氏が [LoCon](https://github.com/KohakuBlueleaf/LoCon) でその有効性を明らかにしたものです。KohakuBlueleaf氏に深く感謝します。
+
+8GB VRAMでもぎりぎり動作するようです。
+
+[学習についての共通ドキュメント](./train_README-ja.md) もあわせてご覧ください。
+
+# 学習できるLoRAの種類
+
+以下の二種類をサポートします。以下は当リポジトリ内の独自の名称です。
+
+1. __LoRA-LierLa__ : (LoRA for __Li__ n __e__ a __r__  __La__ yers、リエラと読みます)
+
+    Linear およびカーネルサイズ 1x1 の Conv2d に適用されるLoRA
+
+2. __LoRA-C3Lier__ : (LoRA for __C__ olutional layers with __3__ x3 Kernel and  __Li__ n __e__ a __r__ layers、セリアと読みます)
+
+    1.に加え、カーネルサイズ 3x3 の Conv2d に適用されるLoRA
+
+LoRA-LierLaに比べ、LoRA-C3Liarは適用される層が増える分、高い精度が期待できるかもしれません。
+
+また学習時は __DyLoRA__ を使用することもできます（後述します）。
+
+## 学習したモデルに関する注意
+
+LoRA-LierLa は、AUTOMATIC1111氏のWeb UIのLoRA機能で使用することができます。
+
+LoRA-C3Liarを使いWeb UIで生成するには、こちらの[WebUI用extension](https://github.com/kohya-ss/sd-webui-additional-networks)を使ってください。
+
+いずれも学習したLoRAのモデルを、Stable Diffusionのモデルにこのリポジトリ内のスクリプトであらかじめマージすることもできます。
+
+cloneofsimo氏のリポジトリ、およびd8ahazard氏の[Dreambooth Extension for Stable-Diffusion-WebUI](https://github.com/d8ahazard/sd_dreambooth_extension)とは、現時点では互換性がありません。いくつかの機能拡張を行っているためです（後述）。
+
+# 学習の手順
+
+あらかじめこのリポジトリのREADMEを参照し、環境整備を行ってください。
+
+## データの準備
+
+[学習データの準備について](./train_README-ja.md) を参照してください。
+
+
+## 学習の実行
+
+`train_network.py`を用います。
+
+`train_network.py`では `--network_module` オプションに、学習対象のモジュール名を指定します。LoRAに対応するのは`network.lora`となりますので、それを指定してください。
+
+なお学習率は通常のDreamBoothやfine tuningよりも高めの、`1e-4`～`1e-3`程度を指定するとよいようです。
+
+以下はコマンドラインの例です。
+
+```
+accelerate launch --num_cpu_threads_per_process 1 train_network.py 
+    --pretrained_model_name_or_path=<.ckptまたは.safetensordまたはDiffusers版モデルのディレクトリ> 
+    --dataset_config=<データ準備で作成した.tomlファイル> 
+    --output_dir=<学習したモデルの出力先フォルダ>  
+    --output_name=<学習したモデル出力時のファイル名> 
+    --save_model_as=safetensors 
+    --prior_loss_weight=1.0 
+    --max_train_steps=400 
+    --learning_rate=1e-4 
+    --optimizer_type="AdamW8bit" 
+    --xformers 
+    --mixed_precision="fp16" 
+    --cache_latents 
+    --gradient_checkpointing
+    --save_every_n_epochs=1 
+    --network_module=networks.lora
+```
+
+このコマンドラインでは LoRA-LierLa が学習されます。
+
+`--output_dir` オプションで指定したフォルダに、LoRAのモデルが保存されます。他のオプション、オプティマイザ等については [学習の共通ドキュメント](./train_README-ja.md) の「よく使われるオプション」も参照してください。
+
+その他、以下のオプションが指定できます。
+
+* `--network_dim`
+  * LoRAのRANKを指定します（``--networkdim=4``など）。省略時は4になります。数が多いほど表現力は増しますが、学習に必要なメモリ、時間は増えます。また闇雲に増やしても良くないようです。
+* `--network_alpha`
+  *  アンダーフローを防ぎ安定して学習するための ``alpha`` 値を指定します。デフォルトは1です。``network_dim``と同じ値を指定すると以前のバージョンと同じ動作になります。
+* `--persistent_data_loader_workers`
+  * Windows環境で指定するとエポック間の待ち時間が大幅に短縮されます。
+* `--max_data_loader_n_workers`
+  * データ読み込みのプロセス数を指定します。プロセス数が多いとデータ読み込みが速くなりGPUを効率的に利用できますが、メインメモリを消費します。デフォルトは「`8` または `CPU同時実行スレッド数-1` の小さいほう」なので、メインメモリに余裕がない場合や、GPU使用率が90%程度以上なら、それらの数値を見ながら `2` または `1` 程度まで下げてください。
+* `--network_weights`
+  * 学習前に学習済みのLoRAの重みを読み込み、そこから追加で学習します。
+* `--network_train_unet_only`
+  * U-Netに関連するLoRAモジュールのみ有効とします。fine tuning的な学習で指定するとよいかもしれません。
+* `--network_train_text_encoder_only`
+  * Text Encoderに関連するLoRAモジュールのみ有効とします。Textual Inversion的な効果が期待できるかもしれません。
+* `--unet_lr`
+  * U-Netに関連するLoRAモジュールに、通常の学習率（--learning_rateオプションで指定）とは異なる学習率を使う時に指定します。
+* `--text_encoder_lr`
+  * Text Encoderに関連するLoRAモジュールに、通常の学習率（--learning_rateオプションで指定）とは異なる学習率を使う時に指定します。Text Encoderのほうを若干低めの学習率（5e-5など）にしたほうが良い、という話もあるようです。
+* `--network_args`
+  * 複数の引数を指定できます。後述します。
+
+`--network_train_unet_only` と `--network_train_text_encoder_only` の両方とも未指定時（デフォルト）はText EncoderとU-Netの両方のLoRAモジュールを有効にします。
+
+# その他の学習方法
+
+## LoRA-C3Lier を学習する
+
+`--network_args` に以下のように指定してください。`conv_dim` で Conv2d (3x3) の rank を、`conv_alpha` で alpha を指定してください。
+
+```
+--network_args "conv_dim=4" "conv_alpha=1"
+```
+
+以下のように alpha 省略時は1になります。
+
+```
+--network_args "conv_dim=4"
+```
+
+## DyLoRA
+
+DyLoRAはこちらの論文で提案されたものです。[DyLoRA: Parameter Efficient Tuning of Pre-trained Models using Dynamic Search-Free Low-Rank Adaptation](https://arxiv.org/abs/2210.07558)　公式実装は[こちら](https://github.com/huawei-noah/KD-NLP/tree/main/DyLoRA)です。
+
+論文によると、LoRAのrankは必ずしも高いほうが良いわけではなく、対象のモデル、データセット、タスクなどにより適切なrankを探す必要があるようです。DyLoRAを使うと、指定したdim(rank)以下のさまざまなrankで同時にLoRAを学習します。これにより最適なrankをそれぞれ学習して探す手間を省くことができます。
+
+当リポジトリの実装は公式実装をベースに独自の拡張を加えています（そのため不具合などあるかもしれません）。
+
+### 当リポジトリのDyLoRAの特徴
+
+学習後のDyLoRAのモデルファイルはLoRAと互換性があります。また、モデルファイルから指定したdim(rank)以下の複数のdimのLoRAを抽出できます。
+
+DyLoRA-LierLa、DyLoRA-C3Lierのどちらも学習できます。
+
+### DyLoRAで学習する
+
+`--network_module=networks.dylora` のように、DyLoRAに対応する`network.dylora`を指定してください。
+
+また `--network_args` に、たとえば`--network_args "unit=4"`のように`unit`を指定します。`unit`はrankを分割する単位です。たとえば`--network_dim=16 --network_args "unit=4"` のように指定します。`unit`は`network_dim`を割り切れる値（`network_dim`は`unit`の倍数）としてください。
+
+`unit`を指定しない場合は、`unit=1`として扱われます。
+
+記述例は以下です。
+
+```
+--network_module=networks.dylora --network_dim=16 --network_args "unit=4"
+
+--network_module=networks.dylora --network_dim=32 --network_alpha=16 --network_args "unit=4"
+```
+
+DyLoRA-C3Lierの場合は、`--network_args` に`"conv_dim=4"`のように`conv_dim`を指定します。通常のLoRAと異なり、`conv_dim`は`network_dim`と同じ値である必要があります。記述例は以下です。
+
+```
+--network_module=networks.dylora --network_dim=16 --network_args "conv_dim=16" "unit=4"
+
+--network_module=networks.dylora --network_dim=32 --network_alpha=16 --network_args "conv_dim=32" "conv_alpha=16" "unit=8"
+```
+
+たとえばdim=16、unit=4（後述）で学習すると、4、8、12、16の4つのrankのLoRAを学習、抽出できます。抽出した各モデルで画像を生成し、比較することで、最適なrankのLoRAを選択できます。
+
+その他のオプションは通常のLoRAと同じです。
+
+※ `unit`は当リポジトリの独自拡張で、DyLoRAでは同dim(rank)の通常LoRAに比べると学習時間が長くなることが予想されるため、分割単位を大きくしたものです。
+
+### DyLoRAのモデルからLoRAモデルを抽出する
+
+`networks`フォルダ内の `extract_lora_from_dylora.py`を使用します。指定した`unit`単位で、DyLoRAのモデルからLoRAのモデルを抽出します。
+
+コマンドラインはたとえば以下のようになります。
+
+```powershell
+python networks\extract_lora_from_dylora.py --model "foldername/dylora-model.safetensors" --save_to "foldername/dylora-model-split.safetensors" --unit 4
+```
+
+`--model` にはDyLoRAのモデルファイルを指定します。`--save_to` には抽出したモデルを保存するファイル名を指定します（rankの数値がファイル名に付加されます）。`--unit` にはDyLoRAの学習時の`unit`を指定します。
+
+## 階層別学習率
+
+詳細は[PR #355](https://github.com/kohya-ss/sd-scripts/pull/355) をご覧ください。
+
+フルモデルの25個のブロックの重みを指定できます。最初のブロックに該当するLoRAは存在しませんが、階層別LoRA適用等との互換性のために25個としています。またconv2d3x3に拡張しない場合も一部のブロックにはLoRAが存在しませんが、記述を統一するため常に25個の値を指定してください。
+
+`--network_args` で以下の引数を指定してください。
+
+- `down_lr_weight` : U-Netのdown blocksの学習率の重みを指定します。以下が指定可能です。
+  - ブロックごとの重み : `"down_lr_weight=0,0,0,0,0,0,1,1,1,1,1,1"` のように12個の数値を指定します。
+  - プリセットからの指定 : `"down_lr_weight=sine"` のように指定します（サインカーブで重みを指定します）。sine, cosine, linear, reverse_linear, zeros が指定可能です。また `"down_lr_weight=cosine+.25"` のように `+数値` を追加すると、指定した数値を加算します（0.25~1.25になります）。
+- `mid_lr_weight` : U-Netのmid blockの学習率の重みを指定します。`"down_lr_weight=0.5"` のように数値を一つだけ指定します。
+- `up_lr_weight` : U-Netのup blocksの学習率の重みを指定します。down_lr_weightと同様です。
+- 指定を省略した部分は1.0として扱われます。また重みを0にするとそのブロックのLoRAモジュールは作成されません。
+- `block_lr_zero_threshold` : 重みがこの値以下の場合、LoRAモジュールを作成しません。デフォルトは0です。
+
+### 階層別学習率コマンドライン指定例:
+
+```powershell
+--network_args "down_lr_weight=0.5,0.5,0.5,0.5,1.0,1.0,1.0,1.0,1.5,1.5,1.5,1.5" "mid_lr_weight=2.0" "up_lr_weight=1.5,1.5,1.5,1.5,1.0,1.0,1.0,1.0,0.5,0.5,0.5,0.5"
+
+--network_args "block_lr_zero_threshold=0.1" "down_lr_weight=sine+.5" "mid_lr_weight=1.5" "up_lr_weight=cosine+.5"
+```
+
+###  階層別学習率tomlファイル指定例:
+
+```toml
+network_args = [ "down_lr_weight=0.5,0.5,0.5,0.5,1.0,1.0,1.0,1.0,1.5,1.5,1.5,1.5", "mid_lr_weight=2.0", "up_lr_weight=1.5,1.5,1.5,1.5,1.0,1.0,1.0,1.0,0.5,0.5,0.5,0.5",]
+
+network_args = [ "block_lr_zero_threshold=0.1", "down_lr_weight=sine+.5", "mid_lr_weight=1.5", "up_lr_weight=cosine+.5", ]
+```
+
+## 階層別dim (rank)
+
+フルモデルの25個のブロックのdim (rank)を指定できます。階層別学習率と同様に一部のブロックにはLoRAが存在しない場合がありますが、常に25個の値を指定してください。
+
+`--network_args` で以下の引数を指定してください。
+
+- `block_dims` : 各ブロックのdim (rank)を指定します。`"block_dims=2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2"` のように25個の数値を指定します。
+- `block_alphas` : 各ブロックのalphaを指定します。block_dimsと同様に25個の数値を指定します。省略時はnetwork_alphaの値が使用されます。
+- `conv_block_dims` : LoRAをConv2d 3x3に拡張し、各ブロックのdim (rank)を指定します。
+- `conv_block_alphas` : LoRAをConv2d 3x3に拡張したときの各ブロックのalphaを指定します。省略時はconv_alphaの値が使用されます。
+
+###  階層別dim (rank)コマンドライン指定例:
+
+```powershell
+--network_args "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2"
+
+--network_args "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2" "conv_block_dims=2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2"
+
+--network_args "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2" "block_alphas=2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2"
+```
+
+###  階層別dim (rank)tomlファイル指定例:
+
+```toml
+network_args = [ "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2",]
+  
+network_args = [ "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2", "block_alphas=2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2",]
+```
+
+# その他のスクリプト
+
+マージ等LoRAに関連するスクリプト群です。
+
+## マージスクリプトについて
+
+merge_lora.pyでStable DiffusionのモデルにLoRAの学習結果をマージしたり、複数のLoRAモデルをマージしたりできます。
+
+### Stable DiffusionのモデルにLoRAのモデルをマージする
+
+マージ後のモデルは通常のStable Diffusionのckptと同様に扱えます。たとえば以下のようなコマンドラインになります。
+
+```
+python networks\merge_lora.py --sd_model ..\model\model.ckpt 
+    --save_to ..\lora_train1\model-char1-merged.safetensors 
+    --models ..\lora_train1\last.safetensors --ratios 0.8
+```
+
+Stable Diffusion v2.xのモデルで学習し、それにマージする場合は、--v2オプションを指定してください。
+
+--sd_modelオプションにマージの元となるStable Diffusionのモデルファイルを指定します（.ckptまたは.safetensorsのみ対応で、Diffusersは今のところ対応していません）。
+
+--save_toオプションにマージ後のモデルの保存先を指定します（.ckptまたは.safetensors、拡張子で自動判定）。
+
+--modelsに学習したLoRAのモデルファイルを指定します。複数指定も可能で、その時は順にマージします。
+
+--ratiosにそれぞれのモデルの適用率（どのくらい重みを元モデルに反映するか）を0~1.0の数値で指定します。例えば過学習に近いような場合は、適用率を下げるとマシになるかもしれません。モデルの数と同じだけ指定してください。
+
+複数指定時は以下のようになります。
+
+```
+python networks\merge_lora.py --sd_model ..\model\model.ckpt 
+    --save_to ..\lora_train1\model-char1-merged.safetensors 
+    --models ..\lora_train1\last.safetensors ..\lora_train2\last.safetensors --ratios 0.8 0.5
+```
+
+### 複数のLoRAのモデルをマージする
+
+__複数のLoRAをマージする場合は原則として `svd_merge_lora.py` を使用してください。__ 単純なup同士やdown同士のマージでは、計算結果が正しくなくなるためです。
+
+`merge_lora.py` によるマージは差分抽出法でLoRAを生成する場合等、ごく限られた場合でのみ有効です。
+
+たとえば以下のようなコマンドラインになります。
+
+```
+python networks\merge_lora.py 
+    --save_to ..\lora_train1\model-char1-style1-merged.safetensors 
+    --models ..\lora_train1\last.safetensors ..\lora_train2\last.safetensors --ratios 0.6 0.4
+```
+
+--sd_modelオプションは指定不要です。
+
+--save_toオプションにマージ後のLoRAモデルの保存先を指定します（.ckptまたは.safetensors、拡張子で自動判定）。
+
+--modelsに学習したLoRAのモデルファイルを指定します。三つ以上も指定可能です。
+
+--ratiosにそれぞれのモデルの比率（どのくらい重みを元モデルに反映するか）を0~1.0の数値で指定します。二つのモデルを一対一でマージす場合は、「0.5 0.5」になります。「1.0 1.0」では合計の重みが大きくなりすぎて、恐らく結果はあまり望ましくないものになると思われます。
+
+v1で学習したLoRAとv2で学習したLoRA、rank（次元数）の異なるLoRAはマージできません。U-NetだけのLoRAとU-Net+Text EncoderのLoRAはマージできるはずですが、結果は未知数です。
+
+
+### その他のオプション
+
+* precision
+  * マージ計算時の精度をfloat、fp16、bf16から指定できます。省略時は精度を確保するためfloatになります。メモリ使用量を減らしたい場合はfp16/bf16を指定してください。
+* save_precision
+  * モデル保存時の精度をfloat、fp16、bf16から指定できます。省略時はprecisionと同じ精度になります。
+
+
+## 複数のrankが異なるLoRAのモデルをマージする
+
+複数のLoRAをひとつのLoRAで近似します（完全な再現はできません）。`svd_merge_lora.py`を用います。たとえば以下のようなコマンドラインになります。
+
+```
+python networks\svd_merge_lora.py 
+    --save_to ..\lora_train1\model-char1-style1-merged.safetensors 
+    --models ..\lora_train1\last.safetensors ..\lora_train2\last.safetensors 
+    --ratios 0.6 0.4 --new_rank 32 --device cuda
+```
+
+`merge_lora.py` と主なオプションは同一です。以下のオプションが追加されています。
+
+- `--new_rank`
+  - 作成するLoRAのrankを指定します。
+- `--new_conv_rank`
+  - 作成する Conv2d 3x3 LoRA の rank を指定します。省略時は `new_rank` と同じになります。
+- `--device`
+  - `--device cuda`としてcudaを指定すると計算をGPU上で行います。処理が速くなります。
+
+## 当リポジトリ内の画像生成スクリプトで生成する
+
+gen_img_diffusers.pyに、--network_module、--network_weightsの各オプションを追加してください。意味は学習時と同様です。
+
+--network_mulオプションで0~1.0の数値を指定すると、LoRAの適用率を変えられます。
+
+## Diffusersのpipelineで生成する
+
+以下の例を参考にしてください。必要なファイルはnetworks/lora.pyのみです。Diffusersのバージョンは0.10.2以外では動作しない可能性があります。
+
+```python
+import torch
+from diffusers import StableDiffusionPipeline
+from networks.lora import LoRAModule, create_network_from_weights
+from safetensors.torch import load_file
+
+# if the ckpt is CompVis based, convert it to Diffusers beforehand with tools/convert_diffusers20_original_sd.py. See --help for more details.
+
+model_id_or_dir = r"model_id_on_hugging_face_or_dir"
+device = "cuda"
+
+# create pipe
+print(f"creating pipe from {model_id_or_dir}...")
+pipe = StableDiffusionPipeline.from_pretrained(model_id_or_dir, revision="fp16", torch_dtype=torch.float16)
+pipe = pipe.to(device)
+vae = pipe.vae
+text_encoder = pipe.text_encoder
+unet = pipe.unet
+
+# load lora networks
+print(f"loading lora networks...")
+
+lora_path1 = r"lora1.safetensors"
+sd = load_file(lora_path1)   # If the file is .ckpt, use torch.load instead.
+network1, sd = create_network_from_weights(0.5, None, vae, text_encoder,unet, sd)
+network1.apply_to(text_encoder, unet)
+network1.load_state_dict(sd)
+network1.to(device, dtype=torch.float16)
+
+# # You can merge weights instead of apply_to+load_state_dict. network.set_multiplier does not work
+# network.merge_to(text_encoder, unet, sd)
+
+lora_path2 = r"lora2.safetensors"
+sd = load_file(lora_path2) 
+network2, sd = create_network_from_weights(0.7, None, vae, text_encoder,unet, sd)
+network2.apply_to(text_encoder, unet)
+network2.load_state_dict(sd)
+network2.to(device, dtype=torch.float16)
+
+lora_path3 = r"lora3.safetensors"
+sd = load_file(lora_path3)
+network3, sd = create_network_from_weights(0.5, None, vae, text_encoder,unet, sd)
+network3.apply_to(text_encoder, unet)
+network3.load_state_dict(sd)
+network3.to(device, dtype=torch.float16)
+
+# prompts
+prompt = "masterpiece, best quality, 1girl, in white shirt, looking at viewer"
+negative_prompt = "bad quality, worst quality, bad anatomy, bad hands"
+
+# exec pipe
+print("generating image...")
+with torch.autocast("cuda"):
+    image = pipe(prompt, guidance_scale=7.5, negative_prompt=negative_prompt).images[0]
+
+# if not merged, you can use set_multiplier
+# network1.set_multiplier(0.8)
+# and generate image again...
+
+# save image
+image.save(r"by_diffusers..png")
+```
+
+## 二つのモデルの差分からLoRAモデルを作成する
+
+[こちらのディスカッション](https://github.com/cloneofsimo/lora/discussions/56)を参考に実装したものです。数式はそのまま使わせていただきました（よく理解していませんが近似には特異値分解を用いるようです）。
+
+二つのモデル（たとえばfine tuningの元モデルとfine tuning後のモデル）の差分を、LoRAで近似します。
+
+### スクリプトの実行方法
+
+以下のように指定してください。
+```
+python networks\extract_lora_from_models.py --model_org base-model.ckpt
+    --model_tuned fine-tuned-model.ckpt 
+    --save_to lora-weights.safetensors --dim 4
+```
+
+--model_orgオプションに元のStable Diffusionモデルを指定します。作成したLoRAモデルを適用する場合は、このモデルを指定して適用することになります。.ckptまたは.safetensorsが指定できます。
+
+--model_tunedオプションに差分を抽出する対象のStable Diffusionモデルを指定します。たとえばfine tuningやDreamBooth後のモデルを指定します。.ckptまたは.safetensorsが指定できます。
+
+--save_toにLoRAモデルの保存先を指定します。--dimにLoRAの次元数を指定します。
+
+生成されたLoRAモデルは、学習したLoRAモデルと同様に使用できます。
+
+Text Encoderが二つのモデルで同じ場合にはLoRAはU-NetのみのLoRAとなります。
+
+### その他のオプション
+
+- `--v2`
+  - v2.xのStable Diffusionモデルを使う場合に指定してください。
+- `--device`
+  - ``--device cuda``としてcudaを指定すると計算をGPU上で行います。処理が速くなります（CPUでもそこまで遅くないため、せいぜい倍～数倍程度のようです）。
+- `--save_precision`
+  - LoRAの保存形式を"float", "fp16", "bf16"から指定します。省略時はfloatになります。
+- `--conv_dim`
+  - 指定するとLoRAの適用範囲を Conv2d 3x3 へ拡大します。Conv2d 3x3 の rank を指定します。
+
+## 画像リサイズスクリプト
+
+（のちほどドキュメントを整理しますがとりあえずここに説明を書いておきます。）
+
+Aspect Ratio Bucketingの機能拡張で、小さな画像については拡大しないでそのまま教師データとすることが可能になりました。元の教師画像を縮小した画像を、教師データに加えると精度が向上したという報告とともに前処理用のスクリプトをいただきましたので整備して追加しました。bmaltais氏に感謝します。
+
+### スクリプトの実行方法
+
+以下のように指定してください。元の画像そのまま、およびリサイズ後の画像が変換先フォルダに保存されます。リサイズ後の画像には、ファイル名に ``+512x512`` のようにリサイズ先の解像度が付け加えられます（画像サイズとは異なります）。リサイズ先の解像度より小さい画像は拡大されることはありません。
+
+```
+python tools\resize_images_to_resolution.py --max_resolution 512x512,384x384,256x256 --save_as_png 
+    --copy_associated_files 元画像フォルダ 変換先フォルダ
+```
+
+元画像フォルダ内の画像ファイルが、指定した解像度（複数指定可）と同じ面積になるようにリサイズされ、変換先フォルダに保存されます。画像以外のファイルはそのままコピーされます。
+
+``--max_resolution`` オプションにリサイズ先のサイズを例のように指定してください。面積がそのサイズになるようにリサイズします。複数指定すると、それぞれの解像度でリサイズされます。``512x512,384x384,256x256``なら、変換先フォルダの画像は、元サイズとリサイズ後サイズ×3の計4枚になります。
+
+``--save_as_png`` オプションを指定するとpng形式で保存します。省略するとjpeg形式（quality=100）で保存されます。
+
+``--copy_associated_files`` オプションを指定すると、拡張子を除き画像と同じファイル名（たとえばキャプションなど）のファイルが、リサイズ後の画像のファイル名と同じ名前でコピーされます。
+
+
+### その他のオプション
+
+- divisible_by
+  - リサイズ後の画像のサイズ（縦、横のそれぞれ）がこの値で割り切れるように、画像中心を切り出します。
+- interpolation
+  - 縮小時の補完方法を指定します。``area, cubic, lanczos4``から選択可能で、デフォルトは``area``です。
+
+
+# 追加情報
+
+## cloneofsimo氏のリポジトリとの違い
+
+2022/12/25時点では、当リポジトリはLoRAの適用個所をText EncoderのMLP、U-NetのFFN、Transformerのin/out projectionに拡大し、表現力が増しています。ただその代わりメモリ使用量は増え、8GBぎりぎりになりました。
+
+またモジュール入れ替え機構は全く異なります。
+
+## 将来拡張について
+
+LoRAだけでなく他の拡張にも対応可能ですので、それらも追加予定です。
diff --git a/docs/train_network_README-zh.md b/docs/train_network_README-zh.md
new file mode 100644
index 0000000000000000000000000000000000000000..ed7a0c4ef730ffaea12220f3678995aa2c944941
--- /dev/null
+++ b/docs/train_network_README-zh.md
@@ -0,0 +1,466 @@
+# 关于LoRA的学习。
+
+[LoRA: Low-Rank Adaptation of Large Language Models](https://arxiv.org/abs/2106.09685)（arxiv）、[LoRA](https://github.com/microsoft/LoRA)（github）这是应用于Stable Diffusion“稳定扩散”的内容。
+
+[cloneofsimo先生的代码仓库](https://github.com/cloneofsimo/lora) 我们非常感謝您提供的参考。非常感謝。
+
+通常情況下，LoRA只适用于Linear和Kernel大小为1x1的Conv2d，但也可以將其擴展到Kernel大小为3x3的Conv2d。
+
+Conv2d 3x3的扩展最初是由 [cloneofsimo先生的代码仓库](https://github.com/cloneofsimo/lora) 
+而KohakuBlueleaf先生在[LoCon](https://github.com/KohakuBlueleaf/LoCon)中揭示了其有效性。我们深深地感谢KohakuBlueleaf先生。
+
+看起来即使在8GB VRAM上也可以勉强运行。
+
+请同时查看关于[学习的通用文档](./train_README-zh.md)。
+# 可学习的LoRA 类型
+
+支持以下两种类型。以下是本仓库中自定义的名称。
+
+1. __LoRA-LierLa__：(用于 __Li__ n __e__ a __r__  __La__ yers 的 LoRA，读作 "Liela")
+
+    适用于 Linear 和卷积层 Conv2d 的 1x1 Kernel 的 LoRA
+
+2. __LoRA-C3Lier__：(用于具有 3x3 Kernel 的卷积层和 __Li__ n __e__ a __r__ 层的 LoRA，读作 "Seria")
+
+    除了第一种类型外，还适用于 3x3 Kernel 的 Conv2d 的 LoRA
+
+与 LoRA-LierLa 相比，LoRA-C3Lier 可能会获得更高的准确性，因为它适用于更多的层。
+
+在训练时，也可以使用 __DyLoRA__（将在后面介绍）。
+
+## 请注意与所学模型相关的事项。
+
+LoRA-LierLa可以用于AUTOMATIC1111先生的Web UI LoRA功能。
+
+要使用LoRA-C3Liar并在Web UI中生成，请使用此处的[WebUI用extension](https://github.com/kohya-ss/sd-webui-additional-networks)。
+
+在此存储库的脚本中，您还可以预先将经过训练的LoRA模型合并到Stable Diffusion模型中。
+
+请注意，与cloneofsimo先生的存储库以及d8ahazard先生的[Stable-Diffusion-WebUI的Dreambooth扩展](https://github.com/d8ahazard/sd_dreambooth_extension)不兼容，因为它们进行了一些功能扩展（如下文所述）。
+
+# 学习步骤
+
+请先参考此存储库的README文件并进行环境设置。
+
+## 准备数据
+
+请参考 [关于准备学习数据](./train_README-zh.md)。
+
+## 网络训练
+
+使用`train_network.py`。
+
+在`train_network.py`中，使用`--network_module`选项指定要训练的模块名称。对于LoRA模块，它应该是`network.lora`，请指定它。
+
+请注意，学习率应该比通常的DreamBooth或fine tuning要高，建议指定为`1e-4`至`1e-3`左右。
+
+以下是命令行示例。
+
+```
+accelerate launch --num_cpu_threads_per_process 1 train_network.py 
+    --pretrained_model_name_or_path=<.ckpt或.safetensord或Diffusers版模型目录> 
+    --dataset_config=<数据集配置的.toml文件> 
+    --output_dir=<训练过程中的模型输出文件夹>  
+    --output_name=<训练模型输出时的文件名> 
+    --save_model_as=safetensors 
+    --prior_loss_weight=1.0 
+    --max_train_steps=400 
+    --learning_rate=1e-4 
+    --optimizer_type="AdamW8bit" 
+    --xformers 
+    --mixed_precision="fp16" 
+    --cache_latents 
+    --gradient_checkpointing
+    --save_every_n_epochs=1 
+    --network_module=networks.lora
+```
+
+在这个命令行中，LoRA-LierLa将会被训练。
+
+LoRA的模型将会被保存在通过`--output_dir`选项指定的文件夹中。关于其他选项和优化器等，请参阅[学习的通用文档](./train_README-zh.md)中的“常用选项”。
+
+此外，还可以指定以下选项：
+
+* `--network_dim`
+  * 指定LoRA的RANK（例如：`--network_dim=4`）。默认值为4。数值越大表示表现力越强，但需要更多的内存和时间来训练。而且不要盲目增加此数值。
+* `--network_alpha`
+  * 指定用于防止下溢并稳定训练的alpha值。默认值为1。如果与`network_dim`指定相同的值，则将获得与以前版本相同的行为。
+* `--persistent_data_loader_workers`
+  * 在Windows环境中指定可大幅缩短epoch之间的等待时间。
+* `--max_data_loader_n_workers`
+  * 指定数据读取进程的数量。进程数越多，数据读取速度越快，可以更有效地利用GPU，但会占用主存。默认值为“`8`或`CPU同步执行线程数-1`的最小值”，因此如果主存不足或GPU使用率超过90％，则应将这些数字降低到约`2`或`1`。
+* `--network_weights`
+  * 在训练之前读取预训练的LoRA权重，并在此基础上进行进一步的训练。
+* `--network_train_unet_only`
+  * 仅启用与U-Net相关的LoRA模块。在类似fine tuning的学习中指定此选项可能会很有用。
+* `--network_train_text_encoder_only`
+  * 仅启用与Text Encoder相关的LoRA模块。可能会期望Textual Inversion效果。
+* `--unet_lr`
+  * 当在U-Net相关的LoRA模块中使用与常规学习率（由`--learning_rate`选项指定）不同的学习率时，应指定此选项。
+* `--text_encoder_lr`
+  * 当在Text Encoder相关的LoRA模块中使用与常规学习率（由`--learning_rate`选项指定）不同的学习率时，应指定此选项。可能最好将Text Encoder的学习率稍微降低（例如5e-5）。
+* `--network_args`
+  * 可以指定多个参数。将在下面详细说明。
+
+当未指定`--network_train_unet_only`和`--network_train_text_encoder_only`时（默认情况），将启用Text Encoder和U-Net的两个LoRA模块。
+
+# 其他的学习方法
+
+## 学习 LoRA-C3Lier
+
+请使用以下方式
+
+```
+--network_args "conv_dim=4"
+```
+
+DyLoRA是在这篇论文中提出的[DyLoRA: Parameter Efficient Tuning of Pre-trained Models using Dynamic Search-Free Low-Rank Adaptation](​https://arxiv.org/abs/2210.07558)，
+[其官方实现可在这里找到](​https://github.com/huawei-noah/KD-NLP/tree/main/DyLoRA)。
+
+根据论文，LoRA的rank并不是越高越好，而是需要根据模型、数据集、任务等因素来寻找合适的rank。使用DyLoRA，可以同时在指定的维度(rank)下学习多种rank的LoRA，从而省去了寻找最佳rank的麻烦。
+
+本存储库的实现基于官方实现进行了自定义扩展（因此可能存在缺陷）。
+
+### 本存储库DyLoRA的特点
+
+DyLoRA训练后的模型文件与LoRA兼容。此外，可以从模型文件中提取多个低于指定维度(rank)的LoRA。
+
+DyLoRA-LierLa和DyLoRA-C3Lier均可训练。
+
+### 使用DyLoRA进行训练
+
+请指定与DyLoRA相对应的`network.dylora`，例如 `--network_module=networks.dylora`。
+
+此外，通过 `--network_args` 指定例如`--network_args "unit=4"`的参数。`unit`是划分rank的单位。例如，可以指定为`--network_dim=16 --network_args "unit=4"`。请将`unit`视为可以被`network_dim`整除的值（`network_dim`是`unit`的倍数）。
+
+如果未指定`unit`，则默认为`unit=1`。
+
+以下是示例说明。
+
+```
+--network_module=networks.dylora --network_dim=16 --network_args "unit=4"
+
+--network_module=networks.dylora --network_dim=32 --network_alpha=16 --network_args "unit=4"
+```
+
+对于DyLoRA-C3Lier，需要在 `--network_args` 中指定 `conv_dim`，例如 `conv_dim=4`。与普通的LoRA不同，`conv_dim`必须与`network_dim`具有相同的值。以下是一个示例描述：
+
+```
+--network_module=networks.dylora --network_dim=16 --network_args "conv_dim=16" "unit=4"
+
+--network_module=networks.dylora --network_dim=32 --network_alpha=16 --network_args "conv_dim=32" "conv_alpha=16" "unit=8"
+```
+
+例如，当使用dim=16、unit=4（如下所述）进行学习时，可以学习和提取4个rank的LoRA，即4、8、12和16。通过在每个提取的模型中生成图像并进行比较，可以选择最佳rank的LoRA。
+
+其他选项与普通的LoRA相同。
+
+*`unit`是本存储库的独有扩展，在DyLoRA中，由于预计相比同维度（rank）的普通LoRA，学习时间更长，因此将分割单位增加。
+
+### 从DyLoRA模型中提取LoRA模型
+
+请使用`networks`文件夹中的`extract_lora_from_dylora.py`。指定`unit`单位后，从DyLoRA模型中提取LoRA模型。
+
+例如，命令行如下：
+
+```powershell
+python networks\extract_lora_from_dylora.py --model "foldername/dylora-model.safetensors" --save_to "foldername/dylora-model-split.safetensors" --unit 4
+```
+
+`--model` 参数用于指定DyLoRA模型文件。`--save_to` 参数用于指定要保存提取的模型的文件名（rank值将附加到文件名中）。`--unit` 参数用于指定DyLoRA训练时的`unit`。 
+
+## 分层学习率
+
+请参阅PR＃355了解详细信息。
+
+您可以指定完整模型的25个块的权重。虽然第一个块没有对应的LoRA，但为了与分层LoRA应用等的兼容性，将其设为25个。此外，如果不扩展到conv2d3x3，则某些块中可能不存在LoRA，但为了统一描述，请始终指定25个值。
+
+请在 `--network_args` 中指定以下参数。
+
+- `down_lr_weight`：指定U-Net down blocks的学习率权重。可以指定以下内容：
+  - 每个块的权重：指定12个数字，例如`"down_lr_weight=0,0,0,0,0,0,1,1,1,1,1,1"`
+  - 从预设中指定：例如`"down_lr_weight=sine"`（使用正弦曲线指定权重）。可以指定sine、cosine、linear、reverse_linear、zeros。另外，添加 `+数字` 时，可以将指定的数字加上（变为0.25〜1.25）。
+- `mid_lr_weight`：指定U-Net mid block的学习率权重。只需指定一个数字，例如 `"mid_lr_weight=0.5"`。
+- `up_lr_weight`：指定U-Net up blocks的学习率权重。与down_lr_weight相同。
+- 省略指定的部分将被视为1.0。另外，如果将权重设为0，则不会创建该块的LoRA模块。
+- `block_lr_zero_threshold`：如果权重小于此值，则不会创建LoRA模块。默认值为0。
+
+### 分层学习率命令行指定示例：
+
+
+```powershell
+--network_args "down_lr_weight=0.5,0.5,0.5,0.5,1.0,1.0,1.0,1.0,1.5,1.5,1.5,1.5" "mid_lr_weight=2.0" "up_lr_weight=1.5,1.5,1.5,1.5,1.0,1.0,1.0,1.0,0.5,0.5,0.5,0.5"
+
+--network_args "block_lr_zero_threshold=0.1" "down_lr_weight=sine+.5" "mid_lr_weight=1.5" "up_lr_weight=cosine+.5"
+```
+
+###  Hierarchical Learning Rate指定的toml文件示例：
+
+```toml
+network_args = [ "down_lr_weight=0.5,0.5,0.5,0.5,1.0,1.0,1.0,1.0,1.5,1.5,1.5,1.5", "mid_lr_weight=2.0", "up_lr_weight=1.5,1.5,1.5,1.5,1.0,1.0,1.0,1.0,0.5,0.5,0.5,0.5",]
+
+network_args = [ "block_lr_zero_threshold=0.1", "down_lr_weight=sine+.5", "mid_lr_weight=1.5", "up_lr_weight=cosine+.5", ]
+```
+
+## 层次结构维度（rank）
+
+您可以指定完整模型的25个块的维度（rank）。与分层学习率一样，某些块可能不存在LoRA，但请始终指定25个值。
+
+请在 `--network_args` 中指定以下参数：
+
+- `block_dims`：指定每个块的维度（rank）。指定25个数字，例如 `"block_dims=2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2"`。
+- `block_alphas`：指定每个块的alpha。与block_dims一样，指定25个数字。如果省略，将使用network_alpha的值。
+- `conv_block_dims`：将LoRA扩展到Conv2d 3x3，并指定每个块的维度（rank）。
+- `conv_block_alphas`：在将LoRA扩展到Conv2d 3x3时指定每个块的alpha。如果省略，将使用conv_alpha的值。
+
+### 层次结构维度（rank）命令行指定示例：
+
+
+```powershell
+--network_args "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2"
+
+--network_args "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2" "conv_block_dims=2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2"
+
+--network_args "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2" "block_alphas=2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2"
+```
+
+### 层级别dim(rank) toml文件指定示例：
+
+```toml
+network_args = [ "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2",]
+  
+network_args = [ "block_dims=2,4,4,4,8,8,8,8,12,12,12,12,16,12,12,12,12,8,8,8,8,4,4,4,2", "block_alphas=2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2",]
+```
+
+# Other scripts
+这些是与LoRA相关的脚本，如合并脚本等。
+
+关于合并脚本
+您可以使用merge_lora.py脚本将LoRA的训练结果合并到稳定扩散模型中，也可以将多个LoRA模型合并。
+
+合并到稳定扩散模型中的LoRA模型
+合并后的模型可以像常规的稳定扩散ckpt一样使用。例如，以下是一个命令行示例：
+
+```
+python networks\merge_lora.py --sd_model ..\model\model.ckpt 
+    --save_to ..\lora_train1\model-char1-merged.safetensors 
+    --models ..\lora_train1\last.safetensors --ratios 0.8
+```
+
+请使用 Stable Diffusion v2.x 模型进行训练并进行合并时，需要指定--v2选项。
+
+使用--sd_model选项指定要合并的 Stable Diffusion 模型文件（仅支持 .ckpt 或 .safetensors 格式，目前不支持 Diffusers）。
+
+使用--save_to选项指定合并后模型的保存路径（根据扩展名自动判断为 .ckpt 或 .safetensors）。
+
+使用--models选项指定已训练的 LoRA 模型文件，也可以指定多个，然后按顺序进行合并。
+
+使用--ratios选项以0~1.0的数字指定每个模型的应用率（将多大比例的权重反映到原始模型中）。例如，在接近过度拟合的情况下，降低应用率可能会使结果更好。请指定与模型数量相同的比率。 
+
+当指定多个模型时，格式如下：
+
+
+```
+python networks\merge_lora.py --sd_model ..\model\model.ckpt 
+    --save_to ..\lora_train1\model-char1-merged.safetensors 
+    --models ..\lora_train1\last.safetensors ..\lora_train2\last.safetensors --ratios 0.8 0.5
+```
+
+### 将多个LoRA模型合并
+
+将多个LoRA模型逐个应用于SD模型与将多个LoRA模型合并后再应用于SD模型之间，由于计算顺序的不同，会得到微妙不同的结果。
+
+例如，下面是一个命令行示例：
+
+```
+python networks\merge_lora.py 
+    --save_to ..\lora_train1\model-char1-style1-merged.safetensors 
+    --models ..\lora_train1\last.safetensors ..\lora_train2\last.safetensors --ratios 0.6 0.4
+```
+
+--sd_model选项不需要指定。
+
+通过--save_to选项指定合并后的LoRA模型的保存位置（.ckpt或.safetensors，根据扩展名自动识别）。
+
+通过--models选项指定学习的LoRA模型文件。可以指定三个或更多。
+
+通过--ratios选项以0~1.0的数字指定每个模型的比率（反映多少权重来自原始模型）。如果将两个模型一对一合并，则比率将是“0.5 0.5”。如果比率为“1.0 1.0”，则总重量将过大，可能会产生不理想的结果。
+
+在v1和v2中学习的LoRA，以及rank（维数）或“alpha”不同的LoRA不能合并。仅包含U-Net的LoRA和包含U-Net+文本编码器的LoRA可以合并，但结果未知。
+
+### 其他选项
+
+* 精度
+  * 可以从float、fp16或bf16中选择合并计算时的精度。默认为float以保证精度。如果想减少内存使用量，请指定fp16/bf16。
+* save_precision
+  * 可以从float、fp16或bf16中选择在保存模型时的精度。默认与精度相同。
+
+## 合并多个维度不同的LoRA模型
+
+将多个LoRA近似为一个LoRA（无法完全复制）。使用'svd_merge_lora.py'。例如，以下是命令行的示例。
+```
+python networks\svd_merge_lora.py 
+    --save_to ..\lora_train1\model-char1-style1-merged.safetensors 
+    --models ..\lora_train1\last.safetensors ..\lora_train2\last.safetensors 
+    --ratios 0.6 0.4 --new_rank 32 --device cuda
+```
+`merge_lora.py`和主要选项相同。以下选项已添加：
+
+- `--new_rank`
+  - 指定要创建的LoRA rank。
+- `--new_conv_rank`
+  - 指定要创建的Conv2d 3x3 LoRA的rank。如果省略，则与`new_rank`相同。
+- `--device`
+  - 如果指定为`--device cuda`，则在GPU上执行计算。处理速度将更快。
+
+## 在此存储库中生成图像的脚本中
+
+请在`gen_img_diffusers.py`中添加`--network_module`和`--network_weights`选项。其含义与训练时相同。
+
+通过`--network_mul`选项，可以指定0~1.0的数字来改变LoRA的应用率。
+
+## 请参考以下示例，在Diffusers的pipeline中生成。
+
+所需文件仅为networks/lora.py。请注意，该示例只能在Diffusers版本0.10.2中正常运行。
+
+```python
+import torch
+from diffusers import StableDiffusionPipeline
+from networks.lora import LoRAModule, create_network_from_weights
+from safetensors.torch import load_file
+
+# if the ckpt is CompVis based, convert it to Diffusers beforehand with tools/convert_diffusers20_original_sd.py. See --help for more details.
+
+model_id_or_dir = r"model_id_on_hugging_face_or_dir"
+device = "cuda"
+
+# create pipe
+print(f"creating pipe from {model_id_or_dir}...")
+pipe = StableDiffusionPipeline.from_pretrained(model_id_or_dir, revision="fp16", torch_dtype=torch.float16)
+pipe = pipe.to(device)
+vae = pipe.vae
+text_encoder = pipe.text_encoder
+unet = pipe.unet
+
+# load lora networks
+print(f"loading lora networks...")
+
+lora_path1 = r"lora1.safetensors"
+sd = load_file(lora_path1)   # If the file is .ckpt, use torch.load instead.
+network1, sd = create_network_from_weights(0.5, None, vae, text_encoder,unet, sd)
+network1.apply_to(text_encoder, unet)
+network1.load_state_dict(sd)
+network1.to(device, dtype=torch.float16)
+
+# # You can merge weights instead of apply_to+load_state_dict. network.set_multiplier does not work
+# network.merge_to(text_encoder, unet, sd)
+
+lora_path2 = r"lora2.safetensors"
+sd = load_file(lora_path2) 
+network2, sd = create_network_from_weights(0.7, None, vae, text_encoder,unet, sd)
+network2.apply_to(text_encoder, unet)
+network2.load_state_dict(sd)
+network2.to(device, dtype=torch.float16)
+
+lora_path3 = r"lora3.safetensors"
+sd = load_file(lora_path3)
+network3, sd = create_network_from_weights(0.5, None, vae, text_encoder,unet, sd)
+network3.apply_to(text_encoder, unet)
+network3.load_state_dict(sd)
+network3.to(device, dtype=torch.float16)
+
+# prompts
+prompt = "masterpiece, best quality, 1girl, in white shirt, looking at viewer"
+negative_prompt = "bad quality, worst quality, bad anatomy, bad hands"
+
+# exec pipe
+print("generating image...")
+with torch.autocast("cuda"):
+    image = pipe(prompt, guidance_scale=7.5, negative_prompt=negative_prompt).images[0]
+
+# if not merged, you can use set_multiplier
+# network1.set_multiplier(0.8)
+# and generate image again...
+
+# save image
+image.save(r"by_diffusers..png")
+```
+
+## 从两个模型的差异中创建LoRA模型。
+
+[参考讨论链接](https://github.com/cloneofsimo/lora/discussions/56)這是參考實現的結果。數學公式沒有改變（我並不完全理解，但似乎使用奇異值分解進行了近似）。
+
+将两个模型（例如微调原始模型和微调后的模型）的差异近似为LoRA。
+
+### 脚本执行方法
+
+请按以下方式指定。
+
+```
+python networks\extract_lora_from_models.py --model_org base-model.ckpt
+    --model_tuned fine-tuned-model.ckpt 
+    --save_to lora-weights.safetensors --dim 4
+```
+
+--model_org 选项指定原始的Stable Diffusion模型。如果要应用创建的LoRA模型，则需要指定该模型并将其应用。可以指定.ckpt或.safetensors文件。
+
+--model_tuned 选项指定要提取差分的目标Stable Diffusion模型。例如，可以指定经过Fine Tuning或DreamBooth后的模型。可以指定.ckpt或.safetensors文件。
+
+--save_to 指定LoRA模型的保存路径。--dim指定LoRA的维数。
+
+生成的LoRA模型可以像已训练的LoRA模型一样使用。
+
+当两个模型的文本编码器相同时，LoRA将成为仅包含U-Net的LoRA。
+
+### 其他选项
+
+- `--v2`
+  - 如果使用v2.x的稳定扩散模型，请指定此选项。
+- `--device`
+  - 指定为 ``--device cuda`` 可在GPU上执行计算。这会使处理速度更快（即使在CPU上也不会太慢，大约快几倍）。
+- `--save_precision`
+  - 指定LoRA的保存格式为“float”、“fp16”、“bf16”。如果省略，将使用float。
+- `--conv_dim`
+  - 指定后，将扩展LoRA的应用范围到Conv2d 3x3。指定Conv2d 3x3的rank。
+  - 
+## 图像大小调整脚本
+
+（稍后将整理文件，但现在先在这里写下说明。）
+
+在 Aspect Ratio Bucketing 的功能扩展中，现在可以将小图像直接用作教师数据，而无需进行放大。我收到了一个用于前处理的脚本，其中包括将原始教师图像缩小的图像添加到教师数据中可以提高准确性的报告。我整理了这个脚本并加入了感谢 bmaltais 先生。
+
+### 执行脚本的方法如下。
+原始图像以及调整大小后的图像将保存到转换目标文件夹中。调整大小后的图像将在文件名中添加“+512x512”之类的调整后的分辨率（与图像大小不同）。小于调整大小后分辨率的图像将不会被放大。
+
+```
+python tools\resize_images_to_resolution.py --max_resolution 512x512,384x384,256x256 --save_as_png 
+    --copy_associated_files 源图像文件夹目标文件夹
+```
+
+在元画像文件夹中的图像文件将被调整大小以达到指定的分辨率（可以指定多个），并保存到目标文件夹中。除图像外的文件将被保留为原样。
+
+请使用“--max_resolution”选项指定调整大小后的大小，使其达到指定的面积大小。如果指定多个，则会在每个分辨率上进行调整大小。例如，“512x512，384x384，256x256”将使目标文件夹中的图像变为原始大小和调整大小后的大小×3共计4张图像。
+
+如果使用“--save_as_png”选项，则会以PNG格式保存。如果省略，则默认以JPEG格式（quality=100）保存。
+
+如果使用“--copy_associated_files”选项，则会将与图像相同的文件名（例如标题等）的文件复制到调整大小后的图像文件的文件名相同的位置，但不包括扩展名。
+
+### 其他选项
+
+- divisible_by
+  - 将图像中心裁剪到能够被该值整除的大小（分别是垂直和水平的大小），以便调整大小后的图像大小可以被该值整除。
+- interpolation
+  - 指定缩小时的插值方法。可从``area、cubic、lanczos4``中选择，默认为``area``。
+
+
+# 追加信息
+
+## 与cloneofsimo的代码库的区别
+
+截至2022年12月25日，本代码库将LoRA应用扩展到了Text Encoder的MLP、U-Net的FFN以及Transformer的输入/输出投影中，从而增强了表现力。但是，内存使用量增加了，接近了8GB的限制。
+
+此外，模块交换机制也完全不同。
+
+## 关于未来的扩展
+
+除了LoRA之外，我们还计划添加其他扩展，以支持更多的功能。
diff --git a/docs/train_ti_README-ja.md b/docs/train_ti_README-ja.md
new file mode 100644
index 0000000000000000000000000000000000000000..86f45a5dcf60331e7551f5da132fa51e2de31ba0
--- /dev/null
+++ b/docs/train_ti_README-ja.md
@@ -0,0 +1,105 @@
+[Textual Inversion](https://textual-inversion.github.io/) の学習についての説明です。
+
+[学習についての共通ドキュメント](./train_README-ja.md) もあわせてご覧ください。
+
+実装に当たっては https://github.com/huggingface/diffusers/tree/main/examples/textual_inversion を大いに参考にしました。
+
+学習したモデルはWeb UIでもそのまま使えます。
+
+# 学習の手順
+
+あらかじめこのリポジトリのREADMEを参照し、環境整備を行ってください。
+
+## データの準備
+
+[学習データの準備について](./train_README-ja.md) を参照してください。
+
+## 学習の実行
+
+``train_textual_inversion.py`` を用います。以下はコマンドラインの例です（DreamBooth手法）。
+
+```
+accelerate launch --num_cpu_threads_per_process 1 train_textual_inversion.py 
+    --dataset_config=<データ準備で作成した.tomlファイル> 
+    --output_dir=<学習したモデルの出力先フォルダ>  
+    --output_name=<学習したモデル出力時のファイル名> 
+    --save_model_as=safetensors 
+    --prior_loss_weight=1.0 
+    --max_train_steps=1600 
+    --learning_rate=1e-6 
+    --optimizer_type="AdamW8bit" 
+    --xformers 
+    --mixed_precision="fp16" 
+    --cache_latents 
+    --gradient_checkpointing
+    --token_string=mychar4 --init_word=cute --num_vectors_per_token=4
+```
+
+``--token_string`` に学習時のトークン文字列を指定します。__学習時のプロンプトは、この文字列を含むようにしてください（token_stringがmychar4なら、``mychar4 1girl`` など）__。プロンプトのこの文字列の部分が、Textual Inversionの新しいtokenに置換されて学習されます。DreamBooth, class+identifier形式のデータセットとして、`token_string` をトークン文字列にするのが最も簡単で確実です。
+
+プロンプトにトークン文字列が含まれているかどうかは、``--debug_dataset`` で置換後のtoken idが表示されますので、以下のように ``49408`` 以降のtokenが存在するかどうかで確認できます。
+
+```
+input ids: tensor([[49406, 49408, 49409, 49410, 49411, 49412, 49413, 49414, 49415, 49407,
+         49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+         49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+         49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+         49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+         49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+         49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+         49407, 49407, 49407, 49407, 49407, 49407, 49407]])
+```
+
+tokenizerがすでに持っている単語（一般的な単語）は使用できません。
+
+``--init_word`` にembeddingsを初期化するときのコピー元トークンの文字列を指定します。学ばせたい概念が近いものを選ぶとよいようです。二つ以上のトークンになる文字列は指定できません。
+
+``--num_vectors_per_token`` にいくつのトークンをこの学習で使うかを指定します。多いほうが表現力が増しますが、その分多くのトークンを消費します。たとえばnum_vectors_per_token=8の場合、指定したトークン文字列は（一般的なプロンプトの77トークン制限のうち）8トークンを消費します。
+
+以上がTextual Inversionのための主なオプションです。以降は他の学習スクリプトと同様です。
+
+`num_cpu_threads_per_process` には通常は1を指定するとよいようです。
+
+`pretrained_model_name_or_path` に追加学習を行う元となるモデルを指定します。Stable Diffusionのcheckpointファイル（.ckptまたは.safetensors）、Diffusersのローカルディスクにあるモデルディレクトリ、DiffusersのモデルID（"stabilityai/stable-diffusion-2"など）が指定できます。
+
+`output_dir` に学習後のモデルを保存するフォルダを指定します。`output_name` にモデルのファイル名を拡張子を除いて指定します。`save_model_as` でsafetensors形式での保存を指定しています。
+
+`dataset_config` に `.toml` ファイルを指定します。ファイル内でのバッチサイズ指定は、当初はメモリ消費を抑えるために `1` としてください。
+
+学習させるステップ数 `max_train_steps` を10000とします。学習率 `learning_rate` はここでは5e-6を指定しています。
+
+省メモリ化のため `mixed_precision="fp16"` を指定します（RTX30 シリーズ以降では `bf16` も指定できます。環境整備時にaccelerateに行った設定と合わせてください）。また `gradient_checkpointing` を指定します。
+
+オプティマイザ（モデルを学習データにあうように最適化＝学習させるクラス）にメモリ消費の少ない 8bit AdamW を使うため、 `optimizer_type="AdamW8bit"` を指定します。
+
+`xformers` オプションを指定し、xformersのCrossAttentionを用います。xformersをインストールしていない場合やエラーとなる場合（環境にもよりますが `mixed_precision="no"` の場合など）、代わりに `mem_eff_attn` オプションを指定すると省メモリ版CrossAttentionを使用します（速度は遅くなります）。
+
+ある程度メモリがある場合は、`.toml` ファイルを編集してバッチサイズをたとえば `8` くらいに増やしてください（高速化と精度向上の可能性があります）。
+
+### よく使われるオプションについて
+
+以下の場合にはオプションに関するドキュメントを参照してください。
+
+- Stable Diffusion 2.xまたはそこからの派生モデルを学習する
+- clip skipを2以上を前提としたモデルを学習する
+- 75トークンを超えたキャプションで学習する
+
+### Textual Inversionでのバッチサイズについて
+
+モデル全体を学習するDreamBoothやfine tuningに比べてメモリ使用量が少ないため、バッチサイズは大きめにできます。
+
+# Textual Inversionのその他の主なオプション
+
+すべてのオプションについては別文書を参照してください。
+
+* `--weights`
+  * 学習前に学習済みのembeddingsを読み込み、そこから追加で学習します。
+* `--use_object_template`
+  * キャプションではなく既定の物体用テンプレート文字列（``a photo of a {}``など）で学習します。公式実装と同じになります。キャプションは無視されます。
+* `--use_style_template`
+  * キャプションではなく既定のスタイル用テンプレート文字列で学習します（``a painting in the style of {}``など）。公式実装と同じになります。キャプションは無視されます。
+
+## 当リポジトリ内の画像生成スクリプトで生成する
+
+gen_img_diffusers.pyに、``--textual_inversion_embeddings`` オプションで学習したembeddingsファイルを指定してください（複数可）。プロンプトでembeddingsファイルのファイル名（拡張子を除く）を使うと、そのembeddingsが適用されます。
+
diff --git a/dreambooth_gui.py b/dreambooth_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd8fa145a5104e041ad3a414779303d65ccf9158
--- /dev/null
+++ b/dreambooth_gui.py
@@ -0,0 +1,939 @@
+# v1: initial release
+# v2: add open and save folder icons
+# v3: Add new Utilities tab for Dreambooth folder preparation
+# v3.1: Adding captionning of images to utilities
+
+import gradio as gr
+import json
+import math
+import os
+import subprocess
+import pathlib
+import argparse
+from datetime import datetime
+from library.common_gui import (
+    get_file_path,
+    get_saveasfile_path,
+    color_aug_changed,
+    save_inference_file,
+    run_cmd_advanced_training,
+    run_cmd_training,
+    update_my_data,
+    check_if_model_exist,
+    output_message,
+    verify_image_folder_pattern,
+    SaveConfigFile,
+    save_to_file
+)
+from library.class_configuration_file import ConfigurationFile
+from library.class_source_model import SourceModel
+from library.class_basic_training import BasicTraining
+from library.class_advanced_training import AdvancedTraining
+from library.class_folders import Folders
+from library.tensorboard_gui import (
+    gradio_tensorboard,
+    start_tensorboard,
+    stop_tensorboard,
+)
+from library.dreambooth_folder_creation_gui import (
+    gradio_dreambooth_folder_creation_tab,
+)
+from library.utilities import utilities_tab
+from library.class_sample_images import SampleImages, run_cmd_sample
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+
+def save_configuration(
+    save_as,
+    file_path,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl,
+    logging_dir,
+    train_data_dir,
+    reg_data_dir,
+    output_dir,
+    max_resolution,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    cache_latents,
+    cache_latents_to_disk,
+    caption_extension,
+    enable_bucket,
+    gradient_checkpointing,
+    full_fp16,
+    no_token_padding,
+    stop_text_encoder_training,
+    # use_8bit_adam,
+    xformers,
+    save_model_as,
+    shuffle_caption,
+    save_state,
+    resume,
+    prior_loss_weight,
+    color_aug,
+    flip_aug,
+    clip_skip,
+    vae,
+    output_name,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    mem_eff_attn,
+    gradient_accumulation_steps,
+    model_list,
+    keep_tokens,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    weighted_captions,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    min_timestep,
+    max_timestep,
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+
+    original_file_path = file_path
+
+    save_as_bool = True if save_as.get('label') == 'True' else False
+
+    if save_as_bool:
+        log.info('Save as...')
+        file_path = get_saveasfile_path(file_path)
+    else:
+        log.info('Save...')
+        if file_path == None or file_path == '':
+            file_path = get_saveasfile_path(file_path)
+
+    if file_path == None or file_path == '':
+        return original_file_path  # In case a file_path was provided and the user decide to cancel the open action
+
+    # Extract the destination directory from the file path
+    destination_directory = os.path.dirname(file_path)
+
+    # Create the destination directory if it doesn't exist
+    if not os.path.exists(destination_directory):
+        os.makedirs(destination_directory)
+
+    SaveConfigFile(parameters=parameters, file_path=file_path, exclusion=['file_path', 'save_as'])
+
+    return file_path
+
+
+def open_configuration(
+    ask_for_file,
+    file_path,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl,
+    logging_dir,
+    train_data_dir,
+    reg_data_dir,
+    output_dir,
+    max_resolution,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    cache_latents,
+    cache_latents_to_disk,
+    caption_extension,
+    enable_bucket,
+    gradient_checkpointing,
+    full_fp16,
+    no_token_padding,
+    stop_text_encoder_training,
+    # use_8bit_adam,
+    xformers,
+    save_model_as,
+    shuffle_caption,
+    save_state,
+    resume,
+    prior_loss_weight,
+    color_aug,
+    flip_aug,
+    clip_skip,
+    vae,
+    output_name,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    mem_eff_attn,
+    gradient_accumulation_steps,
+    model_list,
+    keep_tokens,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    weighted_captions,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    min_timestep,
+    max_timestep,
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+
+    ask_for_file = True if ask_for_file.get('label') == 'True' else False
+
+    original_file_path = file_path
+
+    if ask_for_file:
+        file_path = get_file_path(file_path)
+
+    if not file_path == '' and not file_path == None:
+        # load variables from JSON file
+        with open(file_path, 'r') as f:
+            my_data = json.load(f)
+            log.info('Loading config...')
+            # Update values to fix deprecated use_8bit_adam checkbox and set appropriate optimizer if it is set to True
+            my_data = update_my_data(my_data)
+    else:
+        file_path = original_file_path  # In case a file_path was provided and the user decide to cancel the open action
+        my_data = {}
+
+    values = [file_path]
+    for key, value in parameters:
+        # Set the value in the dictionary to the corresponding value in `my_data`, or the default value if not found
+        if not key in ['ask_for_file', 'file_path']:
+            values.append(my_data.get(key, value))
+    return tuple(values)
+
+
+def train_model(
+    headless,
+    print_only,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl,
+    logging_dir,
+    train_data_dir,
+    reg_data_dir,
+    output_dir,
+    max_resolution,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    cache_latents,
+    cache_latents_to_disk,
+    caption_extension,
+    enable_bucket,
+    gradient_checkpointing,
+    full_fp16,
+    no_token_padding,
+    stop_text_encoder_training_pct,
+    # use_8bit_adam,
+    xformers,
+    save_model_as,
+    shuffle_caption,
+    save_state,
+    resume,
+    prior_loss_weight,
+    color_aug,
+    flip_aug,
+    clip_skip,
+    vae,
+    output_name,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    mem_eff_attn,
+    gradient_accumulation_steps,
+    model_list,  # Keep this. Yes, it is unused here but required given the common list used
+    keep_tokens,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    weighted_captions,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    min_timestep,
+    max_timestep,
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+    
+    print_only_bool = True if print_only.get('label') == 'True' else False
+    log.info(f'Start training Dreambooth...')
+
+    headless_bool = True if headless.get('label') == 'True' else False
+
+    if pretrained_model_name_or_path == '':
+        output_message(
+            msg='Source model information is missing', headless=headless_bool
+        )
+        return
+
+    if train_data_dir == '':
+        output_message(
+            msg='Image folder path is missing', headless=headless_bool
+        )
+        return
+
+    if not os.path.exists(train_data_dir):
+        output_message(
+            msg='Image folder does not exist', headless=headless_bool
+        )
+        return
+
+    if not verify_image_folder_pattern(train_data_dir):
+        return
+
+    if reg_data_dir != '':
+        if not os.path.exists(reg_data_dir):
+            output_message(
+                msg='Regularisation folder does not exist',
+                headless=headless_bool,
+            )
+            return
+
+        if not verify_image_folder_pattern(reg_data_dir):
+            return
+
+    if output_dir == '':
+        output_message(
+            msg='Output folder path is missing', headless=headless_bool
+        )
+        return
+
+    if check_if_model_exist(
+        output_name, output_dir, save_model_as, headless=headless_bool
+    ):
+        return
+
+    if sdxl:
+        output_message(
+            msg='TI training is not compatible with an SDXL model.',
+            headless=headless_bool,
+        )
+        return
+
+    # if optimizer == 'Adafactor' and lr_warmup != '0':
+    #     output_message(
+    #         msg="Warning: lr_scheduler is set to 'Adafactor', so 'LR warmup (% of steps)' will be considered 0.",
+    #         title='Warning',
+    #         headless=headless_bool,
+    #     )
+    #     lr_warmup = '0'
+
+    # Get a list of all subfolders in train_data_dir, excluding hidden folders
+    subfolders = [
+        f
+        for f in os.listdir(train_data_dir)
+        if os.path.isdir(os.path.join(train_data_dir, f))
+        and not f.startswith('.')
+    ]
+
+    # Check if subfolders are present. If not let the user know and return
+    if not subfolders:
+        log.info(
+            f"No {subfolders} were found in train_data_dir can't train..."
+        )
+        return
+
+    total_steps = 0
+
+    # Loop through each subfolder and extract the number of repeats
+    for folder in subfolders:
+        # Extract the number of repeats from the folder name
+        try:
+            repeats = int(folder.split('_')[0])
+        except ValueError:
+            log.info(
+                f"Subfolder {folder} does not have a proper repeat value, please correct the name or remove it... can't train..."
+            )
+            continue
+
+        # Count the number of images in the folder
+        num_images = len(
+            [
+                f
+                for f, lower_f in (
+                    (file, file.lower())
+                    for file in os.listdir(
+                        os.path.join(train_data_dir, folder)
+                    )
+                )
+                if lower_f.endswith(('.jpg', '.jpeg', '.png', '.webp'))
+            ]
+        )
+
+        if num_images == 0:
+            log.info(f'{folder} folder contain no images, skipping...')
+        else:
+            # Calculate the total number of steps for this folder
+            steps = repeats * num_images
+            total_steps += steps
+
+            # Print the result
+            log.info(f'Folder {folder} : steps {steps}')
+
+    if total_steps == 0:
+        log.info(
+            f'No images were found in folder {train_data_dir}... please rectify!'
+        )
+        return
+
+    # Print the result
+    # log.info(f"{total_steps} total steps")
+
+    if reg_data_dir == '':
+        reg_factor = 1
+    else:
+        log.info(
+            f'Regularisation images are used... Will double the number of steps required...'
+        )
+        reg_factor = 2
+
+    # calculate max_train_steps
+    max_train_steps = int(
+        math.ceil(
+            float(total_steps)
+            / int(train_batch_size)
+            / int(gradient_accumulation_steps)
+            * int(epoch)
+            * int(reg_factor)
+        )
+    )
+    log.info(f'max_train_steps = {max_train_steps}')
+
+    # calculate stop encoder training
+    if int(stop_text_encoder_training_pct) == -1:
+        stop_text_encoder_training = -1
+    elif stop_text_encoder_training_pct == None:
+        stop_text_encoder_training = 0
+    else:
+        stop_text_encoder_training = math.ceil(
+            float(max_train_steps) / 100 * int(stop_text_encoder_training_pct)
+        )
+    log.info(f'stop_text_encoder_training = {stop_text_encoder_training}')
+
+    lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+    log.info(f'lr_warmup_steps = {lr_warmup_steps}')
+
+    run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process} "train_db.py"'
+    if v2:
+        run_cmd += ' --v2'
+    if v_parameterization:
+        run_cmd += ' --v_parameterization'
+    if enable_bucket:
+        run_cmd += ' --enable_bucket'
+    if no_token_padding:
+        run_cmd += ' --no_token_padding'
+    if weighted_captions:
+        run_cmd += ' --weighted_captions'
+    run_cmd += (
+        f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"'
+    )
+    run_cmd += f' --train_data_dir="{train_data_dir}"'
+    if len(reg_data_dir):
+        run_cmd += f' --reg_data_dir="{reg_data_dir}"'
+    run_cmd += f' --resolution="{max_resolution}"'
+    run_cmd += f' --output_dir="{output_dir}"'
+    if not logging_dir == '':
+        run_cmd += f' --logging_dir="{logging_dir}"'
+    if not stop_text_encoder_training == 0:
+        run_cmd += (
+            f' --stop_text_encoder_training={stop_text_encoder_training}'
+        )
+    if not save_model_as == 'same as source model':
+        run_cmd += f' --save_model_as={save_model_as}'
+    # if not resume == '':
+    #     run_cmd += f' --resume={resume}'
+    if not float(prior_loss_weight) == 1.0:
+        run_cmd += f' --prior_loss_weight={prior_loss_weight}'
+    if not vae == '':
+        run_cmd += f' --vae="{vae}"'
+    if not output_name == '':
+        run_cmd += f' --output_name="{output_name}"'
+    if int(max_token_length) > 75:
+        run_cmd += f' --max_token_length={max_token_length}'
+    if not max_train_epochs == '':
+        run_cmd += f' --max_train_epochs="{max_train_epochs}"'
+    if not max_data_loader_n_workers == '':
+        run_cmd += (
+            f' --max_data_loader_n_workers="{max_data_loader_n_workers}"'
+        )
+    if int(gradient_accumulation_steps) > 1:
+        run_cmd += f' --gradient_accumulation_steps={int(gradient_accumulation_steps)}'
+
+    run_cmd += run_cmd_training(
+        learning_rate=learning_rate,
+        lr_scheduler=lr_scheduler,
+        lr_warmup_steps=lr_warmup_steps,
+        train_batch_size=train_batch_size,
+        max_train_steps=max_train_steps,
+        save_every_n_epochs=save_every_n_epochs,
+        mixed_precision=mixed_precision,
+        save_precision=save_precision,
+        seed=seed,
+        caption_extension=caption_extension,
+        cache_latents=cache_latents,
+        cache_latents_to_disk=cache_latents_to_disk,
+        optimizer=optimizer,
+        optimizer_args=optimizer_args,
+    )
+
+    run_cmd += run_cmd_advanced_training(
+        max_train_epochs=max_train_epochs,
+        max_data_loader_n_workers=max_data_loader_n_workers,
+        max_token_length=max_token_length,
+        resume=resume,
+        save_state=save_state,
+        mem_eff_attn=mem_eff_attn,
+        clip_skip=clip_skip,
+        flip_aug=flip_aug,
+        color_aug=color_aug,
+        shuffle_caption=shuffle_caption,
+        gradient_checkpointing=gradient_checkpointing,
+        full_fp16=full_fp16,
+        xformers=xformers,
+        keep_tokens=keep_tokens,
+        persistent_data_loader_workers=persistent_data_loader_workers,
+        bucket_no_upscale=bucket_no_upscale,
+        random_crop=random_crop,
+        bucket_reso_steps=bucket_reso_steps,
+        caption_dropout_every_n_epochs=caption_dropout_every_n_epochs,
+        caption_dropout_rate=caption_dropout_rate,
+        noise_offset_type=noise_offset_type,
+        noise_offset=noise_offset,
+        adaptive_noise_scale=adaptive_noise_scale,
+        multires_noise_iterations=multires_noise_iterations,
+        multires_noise_discount=multires_noise_discount,
+        additional_parameters=additional_parameters,
+        vae_batch_size=vae_batch_size,
+        min_snr_gamma=min_snr_gamma,
+        save_every_n_steps=save_every_n_steps,
+        save_last_n_steps=save_last_n_steps,
+        save_last_n_steps_state=save_last_n_steps_state,
+        use_wandb=use_wandb,
+        wandb_api_key=wandb_api_key,
+        scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred,
+        min_timestep=min_timestep,
+        max_timestep=max_timestep,
+    )
+
+    run_cmd += run_cmd_sample(
+        sample_every_n_steps,
+        sample_every_n_epochs,
+        sample_sampler,
+        sample_prompts,
+        output_dir,
+    )
+
+    if print_only_bool:
+        log.warning(
+            'Here is the trainer command as a reference. It will not be executed:\n'
+        )
+        print(run_cmd)
+        
+        save_to_file(run_cmd)
+    else:
+        # Saving config file for model
+        current_datetime = datetime.now()
+        formatted_datetime = current_datetime.strftime("%Y%m%d-%H%M%S")
+        file_path = os.path.join(output_dir, f'{output_name}_{formatted_datetime}.json')
+        
+        log.info(f'Saving training config to {file_path}...')
+
+        SaveConfigFile(parameters=parameters, file_path=file_path, exclusion=['file_path', 'save_as', 'headless', 'print_only'])
+        
+        log.info(run_cmd)
+
+        # Run the command
+        if os.name == 'posix':
+            os.system(run_cmd)
+        else:
+            subprocess.run(run_cmd)
+
+        # check if output_dir/last is a folder... therefore it is a diffuser model
+        last_dir = pathlib.Path(f'{output_dir}/{output_name}')
+
+        if not last_dir.is_dir():
+            # Copy inference model for v2 if required
+            save_inference_file(
+                output_dir, v2, v_parameterization, output_name
+            )
+
+
+def dreambooth_tab(
+    # train_data_dir=gr.Textbox(),
+    # reg_data_dir=gr.Textbox(),
+    # output_dir=gr.Textbox(),
+    # logging_dir=gr.Textbox(),
+    headless=False,
+):
+    dummy_db_true = gr.Label(value=True, visible=False)
+    dummy_db_false = gr.Label(value=False, visible=False)
+    dummy_headless = gr.Label(value=headless, visible=False)
+    
+    with gr.Tab('Training'):
+        gr.Markdown('Train a custom model using kohya dreambooth python code...')
+        
+        # Setup Configuration Files Gradio
+        config = ConfigurationFile(headless)
+        
+        source_model = SourceModel(headless=headless)
+
+        with gr.Tab('Folders'):
+            folders = Folders(headless=headless)
+        with gr.Tab('Parameters'):
+            basic_training = BasicTraining(
+                learning_rate_value='1e-5',
+                lr_scheduler_value='cosine',
+                lr_warmup_value='10',
+            )
+            with gr.Accordion('Advanced Configuration', open=False):
+                advanced_training = AdvancedTraining(headless=headless)
+                advanced_training.color_aug.change(
+                    color_aug_changed,
+                    inputs=[advanced_training.color_aug],
+                    outputs=[basic_training.cache_latents],
+                )
+
+            sample = SampleImages()
+
+        with gr.Tab('Tools'):
+            gr.Markdown(
+                'This section provide Dreambooth tools to help setup your dataset...'
+            )
+            gradio_dreambooth_folder_creation_tab(
+                train_data_dir_input=folders.train_data_dir,
+                reg_data_dir_input=folders.reg_data_dir,
+                output_dir_input=folders.output_dir,
+                logging_dir_input=folders.logging_dir,
+                headless=headless,
+            )
+
+        button_run = gr.Button('Train model', variant='primary')
+
+        button_print = gr.Button('Print training command')
+
+        # Setup gradio tensorboard buttons
+        button_start_tensorboard, button_stop_tensorboard = gradio_tensorboard()
+
+        button_start_tensorboard.click(
+            start_tensorboard,
+            inputs=folders.logging_dir,
+            show_progress=False,
+        )
+
+        button_stop_tensorboard.click(
+            stop_tensorboard,
+            show_progress=False,
+        )
+
+        settings_list = [
+            source_model.pretrained_model_name_or_path,
+            source_model.v2,
+            source_model.v_parameterization,
+            source_model.sdxl_checkbox,
+            folders.logging_dir,
+            folders.train_data_dir,
+            folders.reg_data_dir,
+            folders.output_dir,
+            basic_training.max_resolution,
+            basic_training.learning_rate,
+            basic_training.lr_scheduler,
+            basic_training.lr_warmup,
+            basic_training.train_batch_size,
+            basic_training.epoch,
+            basic_training.save_every_n_epochs,
+            basic_training.mixed_precision,
+            basic_training.save_precision,
+            basic_training.seed,
+            basic_training.num_cpu_threads_per_process,
+            basic_training.cache_latents,
+            basic_training.cache_latents_to_disk,
+            basic_training.caption_extension,
+            basic_training.enable_bucket,
+            advanced_training.gradient_checkpointing,
+            advanced_training.full_fp16,
+            advanced_training.no_token_padding,
+            basic_training.stop_text_encoder_training,
+            advanced_training.xformers,
+            source_model.save_model_as,
+            advanced_training.shuffle_caption,
+            advanced_training.save_state,
+            advanced_training.resume,
+            advanced_training.prior_loss_weight,
+            advanced_training.color_aug,
+            advanced_training.flip_aug,
+            advanced_training.clip_skip,
+            advanced_training.vae,
+            folders.output_name,
+            advanced_training.max_token_length,
+            advanced_training.max_train_epochs,
+            advanced_training.max_data_loader_n_workers,
+            advanced_training.mem_eff_attn,
+            advanced_training.gradient_accumulation_steps,
+            source_model.model_list,
+            advanced_training.keep_tokens,
+            advanced_training.persistent_data_loader_workers,
+            advanced_training.bucket_no_upscale,
+            advanced_training.random_crop,
+            advanced_training.bucket_reso_steps,
+            advanced_training.caption_dropout_every_n_epochs,
+            advanced_training.caption_dropout_rate,
+            basic_training.optimizer,
+            basic_training.optimizer_args,
+            advanced_training.noise_offset_type,
+            advanced_training.noise_offset,
+            advanced_training.adaptive_noise_scale,
+            advanced_training.multires_noise_iterations,
+            advanced_training.multires_noise_discount,
+            sample.sample_every_n_steps,
+            sample.sample_every_n_epochs,
+            sample.sample_sampler,
+            sample.sample_prompts,
+            advanced_training.additional_parameters,
+            advanced_training.vae_batch_size,
+            advanced_training.min_snr_gamma,
+            advanced_training.weighted_captions,
+            advanced_training.save_every_n_steps,
+            advanced_training.save_last_n_steps,
+            advanced_training.save_last_n_steps_state,
+            advanced_training.use_wandb,
+            advanced_training.wandb_api_key,
+            advanced_training.scale_v_pred_loss_like_noise_pred,
+            advanced_training.min_timestep,
+            advanced_training.max_timestep,
+        ]
+
+        config.button_open_config.click(
+            open_configuration,
+            inputs=[dummy_db_true, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name] + settings_list,
+            show_progress=False,
+        )
+
+        config.button_load_config.click(
+            open_configuration,
+            inputs=[dummy_db_false, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name] + settings_list,
+            show_progress=False,
+        )
+
+        config.button_save_config.click(
+            save_configuration,
+            inputs=[dummy_db_false, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name],
+            show_progress=False,
+        )
+
+        config.button_save_as_config.click(
+            save_configuration,
+            inputs=[dummy_db_true, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name],
+            show_progress=False,
+        )
+
+        button_run.click(
+            train_model,
+            inputs=[dummy_headless] + [dummy_db_false] + settings_list,
+            show_progress=False,
+        )
+
+        button_print.click(
+            train_model,
+            inputs=[dummy_headless] + [dummy_db_true] + settings_list,
+            show_progress=False,
+        )
+
+        return (
+            folders.train_data_dir,
+            folders.reg_data_dir,
+            folders.output_dir,
+            folders.logging_dir,
+        )
+
+
+def UI(**kwargs):
+    css = ''
+
+    headless = kwargs.get('headless', False)
+    log.info(f'headless: {headless}')
+
+    if os.path.exists('./style.css'):
+        with open(os.path.join('./style.css'), 'r', encoding='utf8') as file:
+            log.info('Load CSS...')
+            css += file.read() + '\n'
+
+    interface = gr.Blocks(
+        css=css, title='Kohya_ss GUI', theme=gr.themes.Default()
+    )
+
+    with interface:
+        with gr.Tab('Dreambooth'):
+            (
+                train_data_dir_input,
+                reg_data_dir_input,
+                output_dir_input,
+                logging_dir_input,
+            ) = dreambooth_tab(headless=headless)
+        with gr.Tab('Utilities'):
+            utilities_tab(
+                train_data_dir_input=train_data_dir_input,
+                reg_data_dir_input=reg_data_dir_input,
+                output_dir_input=output_dir_input,
+                logging_dir_input=logging_dir_input,
+                enable_copy_info_button=True,
+                headless=headless,
+            )
+
+    # Show the interface
+    launch_kwargs = {}
+    username = kwargs.get('username')
+    password = kwargs.get('password')
+    server_port = kwargs.get('server_port', 0)
+    inbrowser = kwargs.get('inbrowser', False)
+    share = kwargs.get('share', False)
+    server_name = kwargs.get('listen')
+
+    launch_kwargs['server_name'] = server_name
+    if username and password:
+        launch_kwargs['auth'] = (username, password)
+    if server_port > 0:
+        launch_kwargs['server_port'] = server_port
+    if inbrowser:
+        launch_kwargs['inbrowser'] = inbrowser
+    if share:
+        launch_kwargs['share'] = share
+    interface.launch(**launch_kwargs)
+
+
+if __name__ == '__main__':
+    # torch.cuda.set_per_process_memory_fraction(0.48)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--listen',
+        type=str,
+        default='127.0.0.1',
+        help='IP to listen on for connections to Gradio',
+    )
+    parser.add_argument(
+        '--username', type=str, default='', help='Username for authentication'
+    )
+    parser.add_argument(
+        '--password', type=str, default='', help='Password for authentication'
+    )
+    parser.add_argument(
+        '--server_port',
+        type=int,
+        default=0,
+        help='Port to run the server listener on',
+    )
+    parser.add_argument(
+        '--inbrowser', action='store_true', help='Open in browser'
+    )
+    parser.add_argument(
+        '--share', action='store_true', help='Share the gradio UI'
+    )
+    parser.add_argument(
+        '--headless', action='store_true', help='Is the server headless'
+    )
+
+    args = parser.parse_args()
+
+    UI(
+        username=args.username,
+        password=args.password,
+        inbrowser=args.inbrowser,
+        server_port=args.server_port,
+        share=args.share,
+        listen=args.listen,
+        headless=args.headless,
+    )
diff --git a/examples/caption.ps1 b/examples/caption.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..e61f9b31040da4005d041d1549949a32df34d0e1
--- /dev/null
+++ b/examples/caption.ps1
@@ -0,0 +1,14 @@
+# This powershell script will create a text file for each files in the folder
+#
+# Usefull to create base caption that will be augmented on a per image basis
+
+$folder = "D:\some\folder\location\"
+$file_pattern="*.*"
+$caption_text="some caption text"
+
+$files = Get-ChildItem $folder$file_pattern -Include *.png, *.jpg, *.webp -File
+foreach ($file in $files) {
+    if (-not(Test-Path -Path $folder\"$($file.BaseName).txt" -PathType Leaf)) {
+        New-Item -ItemType file -Path $folder -Name "$($file.BaseName).txt" -Value $caption_text
+    }
+}
\ No newline at end of file
diff --git a/examples/caption_subfolders.ps1 b/examples/caption_subfolders.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..0bfba6f01810d7c78270c569fe41831e62bb0137
--- /dev/null
+++ b/examples/caption_subfolders.ps1
@@ -0,0 +1,20 @@
+# This powershell script will create a text file for each files in the folder
+#
+# Usefull to create base caption that will be augmented on a per image basis
+
+$folder = "D:\test\t2\"
+$file_pattern="*.*"
+$text_fir_file="bigeyes style"
+
+foreach ($file in Get-ChildItem $folder\$file_pattern -File) 
+{
+    New-Item -ItemType file -Path $folder -Name "$($file.BaseName).txt" -Value $text_fir_file
+}
+
+foreach($directory in Get-ChildItem -path $folder -Directory)
+{
+    foreach ($file in Get-ChildItem $folder\$directory\$file_pattern) 
+    {
+        New-Item -ItemType file -Path $folder\$directory -Name "$($file.BaseName).txt" -Value $text_fir_file
+    }
+}
diff --git a/examples/kohya-1-folders.ps1 b/examples/kohya-1-folders.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..b2cd3b91be87bb7e808cc494615df922c51c648e
--- /dev/null
+++ b/examples/kohya-1-folders.ps1
@@ -0,0 +1,87 @@
+# This powershell script will create a model using the fine tuning dreambooth method. It will require landscape,
+# portrait and square images.
+#
+# Adjust the script to your own needs
+
+# Sylvia Ritter
+# variable values
+$pretrained_model_name_or_path = "D:\models\v1-5-pruned-mse-vae.ckpt"
+$data_dir = "D:\test\squat"
+$train_dir = "D:\test\"
+$resolution = "512,512"
+
+$image_num = Get-ChildItem $data_dir -Recurse -File -Include *.png | Measure-Object | %{$_.Count}
+
+Write-Output "image_num: $image_num"
+
+$learning_rate = 1e-6
+$dataset_repeats = 40
+$train_batch_size = 8
+$epoch = 1
+$save_every_n_epochs=1
+$mixed_precision="fp16"
+$num_cpu_threads_per_process=6
+
+# You should not have to change values past this point
+
+$output_dir = $train_dir + "\model"
+$repeats = $image_num * $dataset_repeats
+$mts = [Math]::Ceiling($repeats / $train_batch_size * $epoch)
+
+Write-Output "Repeats: $repeats"
+
+.\venv\Scripts\activate
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$pretrained_model_name_or_path `
+    --train_data_dir=$data_dir `
+    --output_dir=$output_dir `
+    --resolution=$resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$mts `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$dataset_repeats `
+    --save_precision="fp16"
+    
+# 2nd pass at half the dataset repeat value
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$output_dir"\last.ckpt" `
+    --train_data_dir=$data_dir `
+    --output_dir=$output_dir"2" `
+    --resolution=$resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$([Math]::Ceiling($mts/2)) `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$([Math]::Ceiling($dataset_repeats/2)) `
+    --save_precision="fp16"
+    
+    accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$output_dir"\last.ckpt" `
+    --train_data_dir=$data_dir `
+    --output_dir=$output_dir"2" `
+    --resolution=$resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$mts `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$dataset_repeats `
+    --save_precision="fp16"
+    
\ No newline at end of file
diff --git a/examples/kohya-3-folders.ps1 b/examples/kohya-3-folders.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..484d6fd5870175715360a2dad0142e4f6d5b23e9
--- /dev/null
+++ b/examples/kohya-3-folders.ps1
@@ -0,0 +1,154 @@
+# This powershell script will create a model using the fine tuning dreambooth method. It will require landscape,
+# portrait and square images.
+#
+# Adjust the script to your own needs
+
+# Sylvia Ritter
+# variable values
+$pretrained_model_name_or_path = "D:\models\v1-5-pruned-mse-vae.ckpt"
+$train_dir = "D:\dreambooth\train_sylvia_ritter\raw_data"
+
+$landscape_image_num = 4
+$portrait_image_num = 25
+$square_image_num = 2
+
+$learning_rate = 1e-6
+$dataset_repeats = 120
+$train_batch_size = 4
+$epoch = 1
+$save_every_n_epochs=1
+$mixed_precision="fp16"
+$num_cpu_threads_per_process=6
+
+$landscape_folder_name = "landscape-pp"
+$landscape_resolution = "832,512"
+$portrait_folder_name = "portrait-pp"
+$portrait_resolution = "448,896"
+$square_folder_name = "square-pp"
+$square_resolution = "512,512"
+
+# You should not have to change values past this point
+
+$landscape_data_dir = $train_dir + "\" + $landscape_folder_name
+$portrait_data_dir = $train_dir + "\" + $portrait_folder_name
+$square_data_dir = $train_dir + "\" + $square_folder_name
+$landscape_output_dir = $train_dir + "\model-l"
+$portrait_output_dir = $train_dir + "\model-lp"
+$square_output_dir = $train_dir + "\model-lps"
+
+$landscape_repeats = $landscape_image_num * $dataset_repeats
+$portrait_repeats = $portrait_image_num * $dataset_repeats
+$square_repeats = $square_image_num * $dataset_repeats
+
+$landscape_mts = [Math]::Ceiling($landscape_repeats / $train_batch_size * $epoch)
+$portrait_mts = [Math]::Ceiling($portrait_repeats / $train_batch_size * $epoch)
+$square_mts = [Math]::Ceiling($square_repeats / $train_batch_size * $epoch)
+
+# Write-Output $landscape_repeats
+
+.\venv\Scripts\activate
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$pretrained_model_name_or_path `
+    --train_data_dir=$landscape_data_dir `
+    --output_dir=$landscape_output_dir `
+    --resolution=$landscape_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$landscape_mts `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$dataset_repeats `
+    --save_precision="fp16"
+    
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$landscape_output_dir"\last.ckpt" `
+    --train_data_dir=$portrait_data_dir `
+    --output_dir=$portrait_output_dir `
+    --resolution=$portrait_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$portrait_mts `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$dataset_repeats `
+    --save_precision="fp16"
+    
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$portrait_output_dir"\last.ckpt" `
+    --train_data_dir=$square_data_dir `
+    --output_dir=$square_output_dir `
+    --resolution=$square_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$square_mts `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$dataset_repeats `
+    --save_precision="fp16"
+    
+# 2nd pass at half the dataset repeat value
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$square_output_dir"\last.ckpt" `
+    --train_data_dir=$landscape_data_dir `
+    --output_dir=$landscape_output_dir"2" `
+    --resolution=$landscape_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$([Math]::Ceiling($landscape_mts/2)) `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$([Math]::Ceiling($dataset_repeats/2)) `
+    --save_precision="fp16"
+    
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$landscape_output_dir"2\last.ckpt" `
+    --train_data_dir=$portrait_data_dir `
+    --output_dir=$portrait_output_dir"2" `
+    --resolution=$portrait_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$([Math]::Ceiling($portrait_mts/2)) `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$([Math]::Ceiling($dataset_repeats/2)) `
+    --save_precision="fp16"
+    
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$portrait_output_dir"2\last.ckpt" `
+    --train_data_dir=$square_data_dir `
+    --output_dir=$square_output_dir"2" `
+    --resolution=$square_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$([Math]::Ceiling($square_mts/2)) `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$([Math]::Ceiling($dataset_repeats/2)) `
+    --save_precision="fp16"
+    
\ No newline at end of file
diff --git a/examples/kohya.ps1 b/examples/kohya.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..874221c367e0e9e8be9652f5315b820fa3915142
--- /dev/null
+++ b/examples/kohya.ps1
@@ -0,0 +1,154 @@
+# This powershell script will create a model using the fine tuning dreambooth method. It will require landscape,
+# portrait and square images.
+#
+# Adjust the script to your own needs
+
+# Sylvia Ritter
+# variable values
+$pretrained_model_name_or_path = "D:\models\v1-5-pruned-mse-vae.ckpt"
+$train_dir = "D:\dreambooth\train_sylvia_ritter\raw_data"
+
+$landscape_image_num = 4
+$portrait_image_num = 25
+$square_image_num = 2
+
+$learning_rate = 1e-6
+$dataset_repeats = 120
+$train_batch_size = 4
+$epoch = 1
+$save_every_n_epochs=1
+$mixed_precision="fp16"
+$num_cpu_threads_per_process=6
+
+$landscape_folder_name = "landscape-pp"
+$landscape_resolution = "832,512"
+$portrait_folder_name = "portrait-pp"
+$portrait_resolution = "448,896"
+$square_folder_name = "square-pp"
+$square_resolution = "512,512"
+
+# You should not have to change values past this point
+
+$landscape_data_dir = $train_dir + "\" + $landscape_folder_name
+$portrait_data_dir = $train_dir + "\" + $portrait_folder_name
+$square_data_dir = $train_dir + "\" + $square_folder_name
+$landscape_output_dir = $train_dir + "\model-l"
+$portrait_output_dir = $train_dir + "\model-lp"
+$square_output_dir = $train_dir + "\model-lps"
+
+$landscape_repeats = $landscape_image_num * $dataset_repeats
+$portrait_repeats = $portrait_image_num * $dataset_repeats
+$square_repeats = $square_image_num * $dataset_repeats
+
+$landscape_mts = [Math]::Ceiling($landscape_repeats / $train_batch_size * $epoch)
+$portrait_mts = [Math]::Ceiling($portrait_repeats / $train_batch_size * $epoch)
+$square_mts = [Math]::Ceiling($square_repeats / $train_batch_size * $epoch)
+
+# Write-Output $landscape_repeats
+
+.\venv\Scripts\activate
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$pretrained_model_name_or_path `
+    --train_data_dir=$landscape_data_dir `
+    --output_dir=$landscape_output_dir `
+    --resolution=$landscape_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$landscape_mts `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$dataset_repeats `
+    --save_half
+    
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$landscape_output_dir"\last.ckpt" `
+    --train_data_dir=$portrait_data_dir `
+    --output_dir=$portrait_output_dir `
+    --resolution=$portrait_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$portrait_mts `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$dataset_repeats `
+    --save_half
+    
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$portrait_output_dir"\last.ckpt" `
+    --train_data_dir=$square_data_dir `
+    --output_dir=$square_output_dir `
+    --resolution=$square_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$square_mts `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$dataset_repeats `
+    --save_half
+    
+# 2nd pass at half the dataset repeat value
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$square_output_dir"\last.ckpt" `
+    --train_data_dir=$landscape_data_dir `
+    --output_dir=$landscape_output_dir"2" `
+    --resolution=$landscape_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$([Math]::Ceiling($landscape_mts/2)) `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$([Math]::Ceiling($dataset_repeats/2)) `
+    --save_half
+    
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$landscape_output_dir"2\last.ckpt" `
+    --train_data_dir=$portrait_data_dir `
+    --output_dir=$portrait_output_dir"2" `
+    --resolution=$portrait_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$([Math]::Ceiling($portrait_mts/2)) `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$([Math]::Ceiling($dataset_repeats/2)) `
+    --save_half
+    
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --pretrained_model_name_or_path=$portrait_output_dir"2\last.ckpt" `
+    --train_data_dir=$square_data_dir `
+    --output_dir=$square_output_dir"2" `
+    --resolution=$square_resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$([Math]::Ceiling($square_mts/2)) `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --cache_latents `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --fine_tuning `
+    --dataset_repeats=$([Math]::Ceiling($dataset_repeats/2)) `
+    --save_half
+    
\ No newline at end of file
diff --git a/examples/kohya_finetune.ps1 b/examples/kohya_finetune.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..4959157719f3106043bf02ca26b84f7e4f6449df
--- /dev/null
+++ b/examples/kohya_finetune.ps1
@@ -0,0 +1,153 @@
+# variables related to the pretrained model
+$pretrained_model_name_or_path = "D:\models\test\samdoesart2\model\last"
+$v2 = 1 # set to 1 for true or 0 for false
+$v_model = 0 # set to 1 for true or 0 for false
+
+# variables related to the training dataset and output directory
+$train_dir = "D:\models\test\samdoesart2"
+$image_folder = "D:\dataset\samdoesart2\raw"
+$output_dir = "D:\models\test\samdoesart2\model_e2\"
+$max_resolution = "512,512"
+
+# variables related to the training process
+$learning_rate = 1e-6
+$lr_scheduler = "constant" # Default is constant
+$lr_warmup = 0 # % of steps to warmup for 0 - 100. Default is 0.
+$dataset_repeats = 40
+$train_batch_size = 8
+$epoch = 1
+$save_every_n_epochs = 1
+$mixed_precision = "bf16"
+$save_precision = "fp16" # use fp16 for better compatibility with auto1111 and other repo
+$seed = "494481440"
+$num_cpu_threads_per_process = 6
+$train_text_encoder = 0 # set to 1 to train text encoder otherwise set to 0
+
+# variables related to the resulting diffuser model. If input is ckpt or tensors then it is not applicable
+$convert_to_safetensors = 1 # set to 1 to convert resulting diffuser to ckpt
+$convert_to_ckpt = 1 # set to 1 to convert resulting diffuser to ckpt
+
+# other variables
+$kohya_finetune_repo_path = "D:\kohya_ss"
+
+### You should not need to change things below
+
+# Set variables to useful values using ternary operator
+$v_model = ($v_model -eq 0) ? $null : "--v_parameterization"
+$v2 = ($v2 -eq 0) ? $null : "--v2"
+$train_text_encoder = ($train_text_encoder -eq 0) ? $null : "--train_text_encoder"
+
+# stop script on error
+$ErrorActionPreference = "Stop"
+
+# define a list of substrings to search for
+$substrings_v2 = "stable-diffusion-2-1-base", "stable-diffusion-2-base"
+
+# check if $v2 and $v_model are empty and if $pretrained_model_name_or_path contains any of the substrings in the v2 list
+if ($v2 -eq $null -and $v_model -eq $null -and ($substrings_v2 | Where-Object { $pretrained_model_name_or_path -match $_ }).Count -gt 0) {
+    Write-Host("SD v2 model detected. Setting --v2 parameter")
+    $v2 = "--v2"
+    $v_model = $null
+}
+
+# define a list of substrings to search for v-objective
+$substrings_v_model = "stable-diffusion-2-1", "stable-diffusion-2"
+
+# check if $v2 and $v_model are empty and if $pretrained_model_name_or_path contains any of the substrings in the v_model list
+elseif ($v2 -eq $null -and $v_model -eq $null -and ($substrings_v_model | Where-Object { $pretrained_model_name_or_path -match $_ }).Count -gt 0) {
+    Write-Host("SD v2 v_model detected. Setting --v2 parameter and --v_parameterization")
+    $v2 = "--v2"
+    $v_model = "--v_parameterization"
+}
+
+# activate venv
+cd $kohya_finetune_repo_path
+.\venv\Scripts\activate
+
+# create caption json file
+if (!(Test-Path -Path $train_dir)) {
+    New-Item -Path $train_dir -ItemType "directory"
+}
+
+python $kohya_finetune_repo_path\script\merge_captions_to_metadata.py `
+    --caption_extention ".txt" $image_folder $train_dir"\meta_cap.json"
+
+# create images buckets
+python $kohya_finetune_repo_path\script\prepare_buckets_latents.py `
+    $image_folder `
+    $train_dir"\meta_cap.json" `
+    $train_dir"\meta_lat.json" `
+    $pretrained_model_name_or_path `
+    --batch_size 4 --max_resolution $max_resolution --mixed_precision $mixed_precision
+
+# Get number of valid images
+$image_num = Get-ChildItem "$image_folder" -Recurse -File -Include *.npz | Measure-Object | % { $_.Count }
+
+$repeats = $image_num * $dataset_repeats
+Write-Host("Repeats = $repeats")
+
+# calculate max_train_set
+$max_train_set = [Math]::Ceiling($repeats / $train_batch_size * $epoch)
+Write-Host("max_train_set = $max_train_set")
+
+$lr_warmup_steps = [Math]::Round($lr_warmup * $max_train_set / 100)
+Write-Host("lr_warmup_steps = $lr_warmup_steps")
+
+Write-Host("$v2 $v_model")
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process $kohya_finetune_repo_path\script\fine_tune.py `
+    $v2 `
+    $v_model `
+    --pretrained_model_name_or_path=$pretrained_model_name_or_path `
+    --in_json $train_dir\meta_lat.json `
+    --train_data_dir="$image_folder" `
+    --output_dir=$output_dir `
+    --train_batch_size=$train_batch_size `
+    --dataset_repeats=$dataset_repeats `
+    --learning_rate=$learning_rate `
+    --lr_scheduler=$lr_scheduler `
+    --lr_warmup_steps=$lr_warmup_steps `
+    --max_train_steps=$max_train_set `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --seed=$seed `
+    $train_text_encoder `
+    --save_precision=$save_precision
+
+# check if $output_dir\last is a directory... therefore it is a diffuser model
+if (Test-Path "$output_dir\last" -PathType Container) {
+    if ($convert_to_ckpt) {
+        Write-Host("Converting diffuser model $output_dir\last to $output_dir\last.ckpt")
+        python "$kohya_finetune_repo_path\tools\convert_diffusers20_original_sd.py" `
+            $output_dir\last `
+            $output_dir\last.ckpt `
+            --$save_precision
+    }
+    if ($convert_to_safetensors) {
+        Write-Host("Converting diffuser model $output_dir\last to $output_dir\last.safetensors")
+        python "$kohya_finetune_repo_path\tools\convert_diffusers20_original_sd.py" `
+            $output_dir\last `
+            $output_dir\last.safetensors `
+            --$save_precision
+    }
+}
+
+# define a list of substrings to search for inference file
+$substrings_sd_model = ".ckpt", ".safetensors"
+$matching_extension = foreach ($ext in $substrings_sd_model) {
+    Get-ChildItem $output_dir -File | Where-Object { $_.Extension -contains $ext }
+}
+
+if ($matching_extension.Count -gt 0) {
+    # copy the file named "v2-inference.yaml" from the "v2_inference" folder to $output_dir as last.yaml
+    if ( $v2 -ne $null -and $v_model -ne $null) {
+        Write-Host("Saving v2-inference-v.yaml as $output_dir\last.yaml")
+        Copy-Item -Path "$kohya_finetune_repo_path\v2_inference\v2-inference-v.yaml" -Destination "$output_dir\last.yaml"
+    }
+    elseif ( $v2 -ne $null ) {
+        Write-Host("Saving v2-inference.yaml as $output_dir\last.yaml")
+        Copy-Item -Path "$kohya_finetune_repo_path\v2_inference\v2-inference.yaml" -Destination "$output_dir\last.yaml"
+    }
+}
\ No newline at end of file
diff --git a/examples/kohya_new-v3.ps1 b/examples/kohya_new-v3.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..2810c37209b8723f7ab5772c2e7461ab08ecb215
--- /dev/null
+++ b/examples/kohya_new-v3.ps1
@@ -0,0 +1,90 @@
+# Sylvia Ritter. AKA: by silvery trait
+
+# variable values
+$pretrained_model_name_or_path = "D:\models\v1-5-pruned-mse-vae.ckpt"
+$train_dir = "D:\dreambooth\train_sylvia_ritter\raw_data"
+$training_folder = "all-images-v3"
+
+$learning_rate = 5e-6
+$dataset_repeats = 40
+$train_batch_size = 6
+$epoch = 4
+$save_every_n_epochs=1
+$mixed_precision="bf16"
+$num_cpu_threads_per_process=6
+
+$max_resolution = "768,576"
+
+# You should not have to change values past this point
+
+# stop script on error
+$ErrorActionPreference = "Stop"
+
+# activate venv
+.\venv\Scripts\activate
+
+# create caption json file
+python D:\kohya_ss\finetune\merge_captions_to_metadata.py `
+--caption_extention ".txt" $train_dir"\"$training_folder $train_dir"\meta_cap.json"
+
+# create images buckets
+python D:\kohya_ss\finetune\prepare_buckets_latents.py `
+    $train_dir"\"$training_folder `
+    $train_dir"\meta_cap.json" `
+    $train_dir"\meta_lat.json" `
+    $pretrained_model_name_or_path `
+    --batch_size 4 --max_resolution $max_resolution --mixed_precision fp16
+
+# Get number of valid images
+$image_num = Get-ChildItem "$train_dir\$training_folder" -Recurse -File -Include *.npz | Measure-Object | %{$_.Count}
+$repeats = $image_num * $dataset_repeats
+
+# calculate max_train_set
+$max_train_set = [Math]::Ceiling($repeats / $train_batch_size * $epoch)
+
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process D:\kohya_ss\finetune\fine_tune.py `
+    --pretrained_model_name_or_path=$pretrained_model_name_or_path `
+    --in_json $train_dir"\meta_lat.json" `
+    --train_data_dir=$train_dir"\"$training_folder `
+    --output_dir=$train_dir"\fine_tuned2" `
+    --train_batch_size=$train_batch_size `
+    --dataset_repeats=$dataset_repeats `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$max_train_set `
+    --use_8bit_adam --xformers `
+    --mixed_precision=$mixed_precision `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --train_text_encoder `
+    --save_precision="fp16"
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process D:\kohya_ss\finetune\fine_tune.py `
+    --pretrained_model_name_or_path=$train_dir"\fine_tuned\last.ckpt" `
+    --in_json $train_dir"\meta_lat.json" `
+    --train_data_dir=$train_dir"\"$training_folder `
+    --output_dir=$train_dir"\fine_tuned2" `
+    --train_batch_size=$train_batch_size `
+    --dataset_repeats=$([Math]::Ceiling($dataset_repeats / 2)) `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$([Math]::Ceiling($max_train_set / 2)) `
+    --use_8bit_adam --xformers `
+    --mixed_precision=$mixed_precision `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --save_precision="fp16"
+
+# Hypernetwork
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process D:\kohya_ss\finetune\fine_tune.py `
+    --pretrained_model_name_or_path=$pretrained_model_name_or_path `
+    --in_json $train_dir"\meta_lat.json" `
+    --train_data_dir=$train_dir"\"$training_folder `
+    --output_dir=$train_dir"\fine_tuned" `
+    --train_batch_size=$train_batch_size `
+    --dataset_repeats=$dataset_repeats `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$max_train_set `
+    --use_8bit_adam --xformers `
+    --mixed_precision=$mixed_precision `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --save_precision="fp16" `
+    --hypernetwork_module="hypernetwork_nai"
\ No newline at end of file
diff --git a/examples/kohya_train_db_fixed_with-reg_SDv2 512 base.ps1 b/examples/kohya_train_db_fixed_with-reg_SDv2 512 base.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..28aa1e70acf48a09644445b45b53530acaa38977
--- /dev/null
+++ b/examples/kohya_train_db_fixed_with-reg_SDv2 512 base.ps1	
@@ -0,0 +1,64 @@
+# This powershell script will create a model using the fine tuning dreambooth method. It will require landscape,
+# portrait and square images.
+#
+# Adjust the script to your own needs
+
+# variable values
+$pretrained_model_name_or_path = "D:\models\512-base-ema.ckpt"
+$data_dir = "D:\models\dariusz_zawadzki\kohya_reg\data"
+$reg_data_dir = "D:\models\dariusz_zawadzki\kohya_reg\reg"
+$logging_dir = "D:\models\dariusz_zawadzki\logs"
+$output_dir = "D:\models\dariusz_zawadzki\train_db_model_reg_v2"
+$resolution = "512,512"
+$lr_scheduler="polynomial"
+$cache_latents = 1 # 1 = true, 0 = false
+
+$image_num = Get-ChildItem $data_dir -Recurse -File -Include *.png, *.jpg, *.webp | Measure-Object | %{$_.Count}
+
+Write-Output "image_num: $image_num"
+
+$dataset_repeats = 200
+$learning_rate = 2e-6
+$train_batch_size = 4
+$epoch = 1
+$save_every_n_epochs=1
+$mixed_precision="bf16"
+$num_cpu_threads_per_process=6
+
+# You should not have to change values past this point
+if ($cache_latents -eq 1) {
+    $cache_latents_value="--cache_latents"
+}
+else {
+    $cache_latents_value=""
+}
+
+$repeats = $image_num * $dataset_repeats
+$mts = [Math]::Ceiling($repeats / $train_batch_size * $epoch)
+
+Write-Output "Repeats: $repeats"
+
+cd D:\kohya_ss
+.\venv\Scripts\activate
+
+accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process train_db.py `
+    --v2 `
+    --pretrained_model_name_or_path=$pretrained_model_name_or_path `
+    --train_data_dir=$data_dir `
+    --output_dir=$output_dir `
+    --resolution=$resolution `
+    --train_batch_size=$train_batch_size `
+    --learning_rate=$learning_rate `
+    --max_train_steps=$mts `
+    --use_8bit_adam `
+    --xformers `
+    --mixed_precision=$mixed_precision `
+    $cache_latents_value `
+    --save_every_n_epochs=$save_every_n_epochs `
+    --logging_dir=$logging_dir `
+    --save_precision="fp16" `
+    --reg_data_dir=$reg_data_dir `
+    --seed=494481440 `
+    --lr_scheduler=$lr_scheduler
+
+# Add the inference yaml file along with the model for proper loading. Need to have the same name as model... Most likelly "last.yaml" in our case.
diff --git a/examples/lucoris extract examples.txt b/examples/lucoris extract examples.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d7cc763d5af1f4e6f542bef71ef32a39a5abecc0
--- /dev/null
+++ b/examples/lucoris extract examples.txt	
@@ -0,0 +1,13 @@
+python tools\lycoris_locon_extract.py --mode quantile --safetensors --linear_ratio 0.9 --conv_ratio 0.9 --device cuda D:/models/v1-5-pruned.ckpt D:/models/cyberrealistic_v12.safetensors "D:/lora/sd1.5/cyberrealistic_v12.safetensors"
+
+python tools\lycoris_locon_extract.py --mode quantile --safetensors --linear_quantile 0.75 --conv_quantile 0.75 --device cuda D:/models/v1-5-pruned.ckpt "C:\Users\berna\Downloads\deliberate_v2.safetensors" "D:/lora/sd1.5/deliberate_v2.safetensors"
+
+python tools\lycoris_locon_extract.py --mode fixed --safetensors --linear_dim 512 --conv_dim 512 --device cuda D:/models/v1-5-pruned.ckpt D:/models/cyberrealistic_v12.safetensors "D:/lora/sd1.5/cyberrealistic_v12.safetensors"
+
+python tools\lycoris_locon_extract.py --use_sparse_bias --sparsity 0.98 --mode quantile --safetensors --linear_quantile 0.75 --conv_quantile 0.75 --device cuda D:/models/v1-5-pruned.ckpt "C:\Users\berna\Downloads\deliberate_v2.safetensors" "D:/lora/sd1.5/deliberate_v2.safetensors"
+
+python tools\lycoris_locon_extract.py --use_sparse_bias --sparsity 0.98 --mode quantile --safetensors --linear_quantile 0.75 --conv_quantile 0.75 --device cuda D:/models/v1-5-pruned.ckpt "D:/models/test\claire_v1.0ee2-000003.safetensors" "D:/lora/sd1.5/claire_v1.0ee2-000003.safetensors"
+
+python tools\lycoris_locon_extract.py --use_sparse_bias --sparsity 0.98 --mode quantile --safetensors --linear_quantile 0.5 --conv_quantile 0.5 --device cuda D:/models/v1-5-pruned.ckpt "D:/models/test\claire_v1.0ee2-000003.safetensors" "D:/lora/sd1.5/claire_v1.0ee2-0.5.safetensors"
+
+python tools\lycoris_locon_extract.py --use_sparse_bias --sparsity 0.98 --mode quantile --safetensors --linear_quantile 0.5 --conv_quantile 0.5 --device cuda D:/models/v1-5-pruned.ckpt "D:/models/test\claire_v1.0f.safetensors" "D:/lora/sd1.5/claire_v1.0f0.5.safetensors"
\ No newline at end of file
diff --git a/examples/pull kohya_ss sd-scripts updates in.md b/examples/pull kohya_ss sd-scripts updates in.md
new file mode 100644
index 0000000000000000000000000000000000000000..47b1c79addd13b4b841f7961bb822ac170484626
--- /dev/null
+++ b/examples/pull kohya_ss sd-scripts updates in.md	
@@ -0,0 +1,32 @@
+## Updating a Local Branch with the Latest sd-scripts Changes
+
+To update your local branch with the most recent changes from kohya/sd-scripts, follow these steps:
+
+1. Add sd-scripts as an alternative remote by executing the following command:
+
+   ```
+   git remote add sd-scripts https://github.com/kohya-ss/sd-scripts.git
+   ```
+
+2. When you wish to perform an update, execute the following commands:
+
+   ```
+   git checkout dev
+   git pull sd-scripts main
+   ```
+
+   Alternatively, if you want to obtain the latest code, even if it may be unstable:
+
+   ```
+   git checkout dev
+   git pull sd-scripts dev
+   ```
+
+3. If you encounter a conflict with the Readme file, you can resolve it by taking the following steps:
+
+   ```
+   git add README.md
+   git merge --continue
+   ```
+
+   This may open a text editor for a commit message, but you can simply save and close it to proceed. Following these steps should resolve the conflict. If you encounter additional merge conflicts, consider them as valuable learning opportunities for personal growth.
\ No newline at end of file
diff --git a/examples/word_frequency.ps1 b/examples/word_frequency.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..ed601d71d00d4d027d5c58517bc6c5e819e16fa9
--- /dev/null
+++ b/examples/word_frequency.ps1
@@ -0,0 +1,15 @@
+$txt_files_folder = "D:\dataset\"
+$txt_prefix_to_ignore = "asds"
+$txt_postfix_ti_ignore = "asds"
+
+# Should not need to touch anything below
+
+# (Get-Content $txt_files_folder"\*.txt" ).Replace(",", "") -Split '\W' | Group-Object -NoElement | Sort-Object -Descending -Property Count
+
+$combined_txt = Get-Content $txt_files_folder"\*.txt"
+$combined_txt = $combined_txt.Replace(",", "")
+$combined_txt = $combined_txt.Replace("$txt_prefix_to_ignore", "")
+$combined_txt = $combined_txt.Replace("$txt_postfix_ti_ignore", "") -Split '\W' | Group-Object -NoElement | Sort-Object -Descending -Property Count
+
+Write-Output "Sorted by count"
+Write-Output $combined_txt.Name
\ No newline at end of file
diff --git a/fine_tune.py b/fine_tune.py
new file mode 100644
index 0000000000000000000000000000000000000000..58a6cda0e11e06b91ad79dce3c23f7235401b2c5
--- /dev/null
+++ b/fine_tune.py
@@ -0,0 +1,476 @@
+# training with captions
+# XXX dropped option: hypernetwork training
+
+import argparse
+import gc
+import math
+import os
+from multiprocessing import Value
+
+from tqdm import tqdm
+import torch
+from accelerate.utils import set_seed
+from diffusers import DDPMScheduler
+
+import library.train_util as train_util
+import library.config_util as config_util
+from library.config_util import (
+    ConfigSanitizer,
+    BlueprintGenerator,
+)
+import library.custom_train_functions as custom_train_functions
+from library.custom_train_functions import (
+    apply_snr_weight,
+    get_weighted_text_embeddings,
+    prepare_scheduler_for_custom_training,
+    pyramid_noise_like,
+    apply_noise_offset,
+    scale_v_prediction_loss_like_noise_prediction,
+)
+
+
+def train(args):
+    train_util.verify_training_args(args)
+    train_util.prepare_dataset_args(args, True)
+
+    cache_latents = args.cache_latents
+
+    if args.seed is not None:
+        set_seed(args.seed)  # 乱数系列を初期化する
+
+    tokenizer = train_util.load_tokenizer(args)
+
+    # データセットを準備する
+    if args.dataset_class is None:
+        blueprint_generator = BlueprintGenerator(ConfigSanitizer(False, True, False, True))
+        if args.dataset_config is not None:
+            print(f"Load dataset config from {args.dataset_config}")
+            user_config = config_util.load_user_config(args.dataset_config)
+            ignored = ["train_data_dir", "in_json"]
+            if any(getattr(args, attr) is not None for attr in ignored):
+                print(
+                    "ignore following options because config file is found: {0} / 設定ファイルが利用されるため以下のオプションは無視されます: {0}".format(
+                        ", ".join(ignored)
+                    )
+                )
+        else:
+            user_config = {
+                "datasets": [
+                    {
+                        "subsets": [
+                            {
+                                "image_dir": args.train_data_dir,
+                                "metadata_file": args.in_json,
+                            }
+                        ]
+                    }
+                ]
+            }
+
+        blueprint = blueprint_generator.generate(user_config, args, tokenizer=tokenizer)
+        train_dataset_group = config_util.generate_dataset_group_by_blueprint(blueprint.dataset_group)
+    else:
+        train_dataset_group = train_util.load_arbitrary_dataset(args, tokenizer)
+
+    current_epoch = Value("i", 0)
+    current_step = Value("i", 0)
+    ds_for_collater = train_dataset_group if args.max_data_loader_n_workers == 0 else None
+    collater = train_util.collater_class(current_epoch, current_step, ds_for_collater)
+
+    if args.debug_dataset:
+        train_util.debug_dataset(train_dataset_group)
+        return
+    if len(train_dataset_group) == 0:
+        print(
+            "No data found. Please verify the metadata file and train_data_dir option. / 画像がありません。メタデータおよびtrain_data_dirオプションを確認してください。"
+        )
+        return
+
+    if cache_latents:
+        assert (
+            train_dataset_group.is_latent_cacheable()
+        ), "when caching latents, either color_aug or random_crop cannot be used / latentをキャッシュするときはcolor_augとrandom_cropは使えません"
+
+    # acceleratorを準備する
+    print("prepare accelerator")
+    accelerator = train_util.prepare_accelerator(args)
+
+    # mixed precisionに対応した型を用意しておき適宜castする
+    weight_dtype, save_dtype = train_util.prepare_dtype(args)
+
+    # モデルを読み込む
+    text_encoder, vae, unet, load_stable_diffusion_format = train_util.load_target_model(args, weight_dtype, accelerator)
+
+    # verify load/save model formats
+    if load_stable_diffusion_format:
+        src_stable_diffusion_ckpt = args.pretrained_model_name_or_path
+        src_diffusers_model_path = None
+    else:
+        src_stable_diffusion_ckpt = None
+        src_diffusers_model_path = args.pretrained_model_name_or_path
+
+    if args.save_model_as is None:
+        save_stable_diffusion_format = load_stable_diffusion_format
+        use_safetensors = args.use_safetensors
+    else:
+        save_stable_diffusion_format = args.save_model_as.lower() == "ckpt" or args.save_model_as.lower() == "safetensors"
+        use_safetensors = args.use_safetensors or ("safetensors" in args.save_model_as.lower())
+
+    # Diffusers版のxformers使用フラグを設定する関数
+    def set_diffusers_xformers_flag(model, valid):
+        #   model.set_use_memory_efficient_attention_xformers(valid)            # 次のリリースでなくなりそう
+        # pipeが自動で再帰的にset_use_memory_efficient_attention_xformersを探すんだって(;´Д｀)
+        # U-Netだけ使う時にはどうすればいいのか……仕方ないからコピって使うか
+        # 0.10.2でなんか巻き戻って個別に指定するようになった(;^ω^)
+
+        # Recursively walk through all the children.
+        # Any children which exposes the set_use_memory_efficient_attention_xformers method
+        # gets the message
+        def fn_recursive_set_mem_eff(module: torch.nn.Module):
+            if hasattr(module, "set_use_memory_efficient_attention_xformers"):
+                module.set_use_memory_efficient_attention_xformers(valid)
+
+            for child in module.children():
+                fn_recursive_set_mem_eff(child)
+
+        fn_recursive_set_mem_eff(model)
+
+    # モデルに xformers とか memory efficient attention を組み込む
+    if args.diffusers_xformers:
+        accelerator.print("Use xformers by Diffusers")
+        set_diffusers_xformers_flag(unet, True)
+    else:
+        # Windows版のxformersはfloatで学習できないのでxformersを使わない設定も可能にしておく必要がある
+        accelerator.print("Disable Diffusers' xformers")
+        set_diffusers_xformers_flag(unet, False)
+        train_util.replace_unet_modules(unet, args.mem_eff_attn, args.xformers, args.sdpa)
+
+    # 学習を準備する
+    if cache_latents:
+        vae.to(accelerator.device, dtype=weight_dtype)
+        vae.requires_grad_(False)
+        vae.eval()
+        with torch.no_grad():
+            train_dataset_group.cache_latents(vae, args.vae_batch_size, args.cache_latents_to_disk, accelerator.is_main_process)
+        vae.to("cpu")
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        gc.collect()
+
+        accelerator.wait_for_everyone()
+
+    # 学習を準備する：モデルを適切な状態にする
+    training_models = []
+    if args.gradient_checkpointing:
+        unet.enable_gradient_checkpointing()
+    training_models.append(unet)
+
+    if args.train_text_encoder:
+        accelerator.print("enable text encoder training")
+        if args.gradient_checkpointing:
+            text_encoder.gradient_checkpointing_enable()
+        training_models.append(text_encoder)
+    else:
+        text_encoder.to(accelerator.device, dtype=weight_dtype)
+        text_encoder.requires_grad_(False)  # text encoderは学習しない
+        if args.gradient_checkpointing:
+            text_encoder.gradient_checkpointing_enable()
+            text_encoder.train()  # required for gradient_checkpointing
+        else:
+            text_encoder.eval()
+
+    if not cache_latents:
+        vae.requires_grad_(False)
+        vae.eval()
+        vae.to(accelerator.device, dtype=weight_dtype)
+
+    for m in training_models:
+        m.requires_grad_(True)
+    params = []
+    for m in training_models:
+        params.extend(m.parameters())
+    params_to_optimize = params
+
+    # 学習に必要なクラスを準備する
+    accelerator.print("prepare optimizer, data loader etc.")
+    _, _, optimizer = train_util.get_optimizer(args, trainable_params=params_to_optimize)
+
+    # dataloaderを準備する
+    # DataLoaderのプロセス数：0はメインプロセスになる
+    n_workers = min(args.max_data_loader_n_workers, os.cpu_count() - 1)  # cpu_count-1 ただし最大で指定された数まで
+    train_dataloader = torch.utils.data.DataLoader(
+        train_dataset_group,
+        batch_size=1,
+        shuffle=True,
+        collate_fn=collater,
+        num_workers=n_workers,
+        persistent_workers=args.persistent_data_loader_workers,
+    )
+
+    # 学習ステップ数を計算する
+    if args.max_train_epochs is not None:
+        args.max_train_steps = args.max_train_epochs * math.ceil(
+            len(train_dataloader) / accelerator.num_processes / args.gradient_accumulation_steps
+        )
+        accelerator.print(f"override steps. steps for {args.max_train_epochs} epochs is / 指定エポックまでのステップ数: {args.max_train_steps}")
+
+    # データセット側にも学習ステップを送信
+    train_dataset_group.set_max_train_steps(args.max_train_steps)
+
+    # lr schedulerを用意する
+    lr_scheduler = train_util.get_scheduler_fix(args, optimizer, accelerator.num_processes)
+
+    # 実験的機能：勾配も含めたfp16学習を行う　モデル全体をfp16にする
+    if args.full_fp16:
+        assert (
+            args.mixed_precision == "fp16"
+        ), "full_fp16 requires mixed precision='fp16' / full_fp16を使う場合はmixed_precision='fp16'を指定してください。"
+        accelerator.print("enable full fp16 training.")
+        unet.to(weight_dtype)
+        text_encoder.to(weight_dtype)
+
+    # acceleratorがなんかよろしくやってくれるらしい
+    if args.train_text_encoder:
+        unet, text_encoder, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+            unet, text_encoder, optimizer, train_dataloader, lr_scheduler
+        )
+    else:
+        unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(unet, optimizer, train_dataloader, lr_scheduler)
+
+    # transform DDP after prepare
+    text_encoder, unet = train_util.transform_if_model_is_DDP(text_encoder, unet)
+
+    # 実験的機能：勾配も含めたfp16学習を行う　PyTorchにパッチを当ててfp16でのgrad scaleを有効にする
+    if args.full_fp16:
+        train_util.patch_accelerator_for_fp16_training(accelerator)
+
+    # resumeする
+    train_util.resume_from_local_or_hf_if_specified(accelerator, args)
+
+    # epoch数を計算する
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+    if (args.save_n_epoch_ratio is not None) and (args.save_n_epoch_ratio > 0):
+        args.save_every_n_epochs = math.floor(num_train_epochs / args.save_n_epoch_ratio) or 1
+
+    # 学習する
+    total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
+    accelerator.print("running training / 学習開始")
+    accelerator.print(f"  num examples / サンプル数: {train_dataset_group.num_train_images}")
+    accelerator.print(f"  num batches per epoch / 1epochのバッチ数: {len(train_dataloader)}")
+    accelerator.print(f"  num epochs / epoch数: {num_train_epochs}")
+    accelerator.print(f"  batch size per device / バッチサイズ: {args.train_batch_size}")
+    accelerator.print(
+        f"  total train batch size (with parallel & distributed & accumulation) / 総バッチサイズ（並列学習、勾配合計含む）: {total_batch_size}"
+    )
+    accelerator.print(f"  gradient accumulation steps / 勾配を合計するステップ数 = {args.gradient_accumulation_steps}")
+    accelerator.print(f"  total optimization steps / 学習ステップ数: {args.max_train_steps}")
+
+    progress_bar = tqdm(range(args.max_train_steps), smoothing=0, disable=not accelerator.is_local_main_process, desc="steps")
+    global_step = 0
+
+    noise_scheduler = DDPMScheduler(
+        beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, clip_sample=False
+    )
+    prepare_scheduler_for_custom_training(noise_scheduler, accelerator.device)
+
+    if accelerator.is_main_process:
+        accelerator.init_trackers("finetuning" if args.log_tracker_name is None else args.log_tracker_name)
+
+    for epoch in range(num_train_epochs):
+        accelerator.print(f"\nepoch {epoch+1}/{num_train_epochs}")
+        current_epoch.value = epoch + 1
+
+        for m in training_models:
+            m.train()
+
+        loss_total = 0
+        for step, batch in enumerate(train_dataloader):
+            current_step.value = global_step
+            with accelerator.accumulate(training_models[0]):  # 複数モデルに対応していない模様だがとりあえずこうしておく
+                with torch.no_grad():
+                    if "latents" in batch and batch["latents"] is not None:
+                        latents = batch["latents"].to(accelerator.device)  # .to(dtype=weight_dtype)
+                    else:
+                        # latentに変換
+                        latents = vae.encode(batch["images"].to(dtype=weight_dtype)).latent_dist.sample()
+                    latents = latents * 0.18215
+                b_size = latents.shape[0]
+
+                with torch.set_grad_enabled(args.train_text_encoder):
+                    # Get the text embedding for conditioning
+                    if args.weighted_captions:
+                        encoder_hidden_states = get_weighted_text_embeddings(
+                            tokenizer,
+                            text_encoder,
+                            batch["captions"],
+                            accelerator.device,
+                            args.max_token_length // 75 if args.max_token_length else 1,
+                            clip_skip=args.clip_skip,
+                        )
+                    else:
+                        input_ids = batch["input_ids"].to(accelerator.device)
+                        encoder_hidden_states = train_util.get_hidden_states(
+                            args, input_ids, tokenizer, text_encoder, None if not args.full_fp16 else weight_dtype
+                        )
+
+                # Sample noise, sample a random timestep for each image, and add noise to the latents,
+                # with noise offset and/or multires noise if specified
+                noise, noisy_latents, timesteps = train_util.get_noise_noisy_latents_and_timesteps(args, noise_scheduler, latents)
+
+                # Predict the noise residual
+                with accelerator.autocast():
+                    noise_pred = unet(noisy_latents, timesteps, encoder_hidden_states).sample
+
+                if args.v_parameterization:
+                    # v-parameterization training
+                    target = noise_scheduler.get_velocity(latents, noise, timesteps)
+                else:
+                    target = noise
+
+                if args.min_snr_gamma or args.scale_v_pred_loss_like_noise_pred:
+                    # do not mean over batch dimension for snr weight or scale v-pred loss
+                    loss = torch.nn.functional.mse_loss(noise_pred.float(), target.float(), reduction="none")
+                    loss = loss.mean([1, 2, 3])
+
+                    if args.min_snr_gamma:
+                        loss = apply_snr_weight(loss, timesteps, noise_scheduler, args.min_snr_gamma)
+                    if args.scale_v_pred_loss_like_noise_pred:
+                        loss = scale_v_prediction_loss_like_noise_prediction(loss, timesteps, noise_scheduler)
+
+                    loss = loss.mean()  # mean over batch dimension
+                else:
+                    loss = torch.nn.functional.mse_loss(noise_pred.float(), target.float(), reduction="mean")
+
+                accelerator.backward(loss)
+                if accelerator.sync_gradients and args.max_grad_norm != 0.0:
+                    params_to_clip = []
+                    for m in training_models:
+                        params_to_clip.extend(m.parameters())
+                    accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
+
+                optimizer.step()
+                lr_scheduler.step()
+                optimizer.zero_grad(set_to_none=True)
+
+            # Checks if the accelerator has performed an optimization step behind the scenes
+            if accelerator.sync_gradients:
+                progress_bar.update(1)
+                global_step += 1
+
+                train_util.sample_images(
+                    accelerator, args, None, global_step, accelerator.device, vae, tokenizer, text_encoder, unet
+                )
+
+                # 指定ステップごとにモデルを保存
+                if args.save_every_n_steps is not None and global_step % args.save_every_n_steps == 0:
+                    accelerator.wait_for_everyone()
+                    if accelerator.is_main_process:
+                        src_path = src_stable_diffusion_ckpt if save_stable_diffusion_format else src_diffusers_model_path
+                        train_util.save_sd_model_on_epoch_end_or_stepwise(
+                            args,
+                            False,
+                            accelerator,
+                            src_path,
+                            save_stable_diffusion_format,
+                            use_safetensors,
+                            save_dtype,
+                            epoch,
+                            num_train_epochs,
+                            global_step,
+                            accelerator.unwrap_model(text_encoder),
+                            accelerator.unwrap_model(unet),
+                            vae,
+                        )
+
+            current_loss = loss.detach().item()  # 平均なのでbatch sizeは関係ないはず
+            if args.logging_dir is not None:
+                logs = {"loss": current_loss, "lr": float(lr_scheduler.get_last_lr()[0])}
+                if (
+                    args.optimizer_type.lower().startswith("DAdapt".lower()) or args.optimizer_type.lower() == "Prodigy".lower()
+                ):  # tracking d*lr value
+                    logs["lr/d*lr"] = (
+                        lr_scheduler.optimizers[0].param_groups[0]["d"] * lr_scheduler.optimizers[0].param_groups[0]["lr"]
+                    )
+                accelerator.log(logs, step=global_step)
+
+            # TODO moving averageにする
+            loss_total += current_loss
+            avr_loss = loss_total / (step + 1)
+            logs = {"loss": avr_loss}  # , "lr": lr_scheduler.get_last_lr()[0]}
+            progress_bar.set_postfix(**logs)
+
+            if global_step >= args.max_train_steps:
+                break
+
+        if args.logging_dir is not None:
+            logs = {"loss/epoch": loss_total / len(train_dataloader)}
+            accelerator.log(logs, step=epoch + 1)
+
+        accelerator.wait_for_everyone()
+
+        if args.save_every_n_epochs is not None:
+            if accelerator.is_main_process:
+                src_path = src_stable_diffusion_ckpt if save_stable_diffusion_format else src_diffusers_model_path
+                train_util.save_sd_model_on_epoch_end_or_stepwise(
+                    args,
+                    True,
+                    accelerator,
+                    src_path,
+                    save_stable_diffusion_format,
+                    use_safetensors,
+                    save_dtype,
+                    epoch,
+                    num_train_epochs,
+                    global_step,
+                    accelerator.unwrap_model(text_encoder),
+                    accelerator.unwrap_model(unet),
+                    vae,
+                )
+
+        train_util.sample_images(accelerator, args, epoch + 1, global_step, accelerator.device, vae, tokenizer, text_encoder, unet)
+
+    is_main_process = accelerator.is_main_process
+    if is_main_process:
+        unet = accelerator.unwrap_model(unet)
+        text_encoder = accelerator.unwrap_model(text_encoder)
+
+    accelerator.end_training()
+
+    if args.save_state and is_main_process:
+        train_util.save_state_on_train_end(args, accelerator)
+
+    del accelerator  # この後メモリを使うのでこれは消す
+
+    if is_main_process:
+        src_path = src_stable_diffusion_ckpt if save_stable_diffusion_format else src_diffusers_model_path
+        train_util.save_sd_model_on_train_end(
+            args, src_path, save_stable_diffusion_format, use_safetensors, save_dtype, epoch, global_step, text_encoder, unet, vae
+        )
+        print("model saved.")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    train_util.add_sd_models_arguments(parser)
+    train_util.add_dataset_arguments(parser, False, True, True)
+    train_util.add_training_arguments(parser, False)
+    train_util.add_sd_saving_arguments(parser)
+    train_util.add_optimizer_arguments(parser)
+    config_util.add_config_arguments(parser)
+    custom_train_functions.add_custom_train_arguments(parser)
+
+    parser.add_argument("--diffusers_xformers", action="store_true", help="use xformers by diffusers / Diffusersでxformersを使用する")
+    parser.add_argument("--train_text_encoder", action="store_true", help="train text encoder / text encoderも学習する")
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    args = train_util.read_config_from_file(args, parser)
+
+    train(args)
diff --git a/fine_tune_README.md b/fine_tune_README.md
new file mode 100644
index 0000000000000000000000000000000000000000..7ffd05d4ab7bd2532c69d68f1166b87607724f78
--- /dev/null
+++ b/fine_tune_README.md
@@ -0,0 +1,465 @@
+It is a fine tuning that corresponds to NovelAI's proposed learning method, automatic captioning, tagging, Windows + VRAM 12GB (for v1.4/1.5) environment, etc.
+
+## overview
+Fine tuning of U-Net of Stable Diffusion using Diffusers. It corresponds to the following improvements in NovelAI's article (For Aspect Ratio Bucketing, I referred to NovelAI's code, but the final code is all original).
+
+* Use the output of the penultimate layer instead of the last layer of CLIP (Text Encoder).
+* Learning at non-square resolutions (Aspect Ratio Bucketing).
+* Extend token length from 75 to 225.
+* Captioning with BLIP (automatic creation of captions), automatic tagging with DeepDanbooru or WD14Tagger.
+* Also supports Hypernetwork learning.
+* Supports Stable Diffusion v2.0 (base and 768/v).
+* By acquiring the output of VAE in advance and saving it to disk, we aim to save memory and speed up learning.
+
+Text Encoder is not trained by default. For fine tuning of the whole model, it seems common to learn only U-Net (NovelAI seems to be the same). Text Encoder can also be learned as an option.
+
+## Additional features
+### Change CLIP output
+CLIP (Text Encoder) converts the text into features in order to reflect the prompt in the image. Stable diffusion uses the output of the last layer of CLIP, but you can change it to use the output of the penultimate layer. According to NovelAI, this will reflect prompts more accurately.
+It is also possible to use the output of the last layer as is.
+*Stable Diffusion 2.0 uses the penultimate layer by default. Do not specify the clip_skip option.
+
+### Training in non-square resolutions
+Stable Diffusion is trained at 512\*512, but also at resolutions such as 256\*1024 and 384\*640. It is expected that this will reduce the cropped portion and learn the relationship between prompts and images more correctly.
+The learning resolution is adjusted vertically and horizontally in units of 64 pixels within a range that does not exceed the resolution area (= memory usage) given as a parameter.
+
+In machine learning, it is common to unify all input sizes, but there are no particular restrictions, and in fact it is okay as long as they are unified within the same batch. NovelAI's bucketing seems to refer to classifying training data in advance for each learning resolution according to the aspect ratio. And by creating a batch with the images in each bucket, the image size of the batch is unified.
+
+### Extending token length from 75 to 225
+Stable diffusion has a maximum of 75 tokens (77 tokens including the start and end), but we will extend it to 225 tokens.
+However, the maximum length that CLIP accepts is 75 tokens, so in the case of 225 tokens, we simply divide it into thirds, call CLIP, and then concatenate the results.
+
+*I'm not sure if this is the preferred implementation. It seems to be working for now. Especially in 2.0, there is no implementation that can be used as a reference, so I have implemented it independently.
+
+*Automatic1111's Web UI seems to divide the text with commas in mind, but in my case, it's a simple division.
+
+## Environmental arrangement
+
+See the [README](./README-en.md) in this repository.
+
+## Preparing teacher data
+
+Prepare the image data you want to learn and put it in any folder. No prior preparation such as resizing is required.
+However, for images that are smaller than the training resolution, it is recommended to enlarge them while maintaining the quality using super-resolution.
+
+It also supports multiple teacher data folders. Preprocessing will be executed for each folder.
+
+For example, store an image like this:
+
+![Teacher data folder screenshot](https://user-images.githubusercontent.com/52813779/208907739-8e89d5fa-6ca8-4b60-8927-f484d2a9ae04.png)
+
+## Automatic captioning
+Skip if you just want to learn tags without captions.
+
+Also, when preparing captions manually, prepare them in the same directory as the teacher data image, with the same file name, extension .caption, etc. Each file should be a text file with only one line.
+
+### Captioning with BLIP
+
+The latest version no longer requires BLIP downloads, weight downloads, and additional virtual environments. Works as-is.
+
+Run make_captions.py in the finetune folder.
+
+```
+python finetune\make_captions.py --batch_size <batch size> <teacher data folder>
+```
+
+If the batch size is 8 and the training data is placed in the parent folder train_data, it will be as follows.
+
+```
+python finetune\make_captions.py --batch_size 8 ..\train_data
+```
+
+A caption file is created in the same directory as the teacher data image with the same file name and extension .caption.
+
+Increase or decrease batch_size according to the VRAM capacity of the GPU. Bigger is faster (I think 12GB of VRAM can be a little more).
+You can specify the maximum length of the caption with the max_length option. Default is 75. It may be longer if the model is trained with a token length of 225.
+You can change the caption extension with the caption_extension option. Default is .caption (.txt conflicts with DeepDanbooru described later).
+
+If there are multiple teacher data folders, execute for each folder.
+
+Note that the inference is random, so the results will change each time you run it. If you want to fix it, specify a random number seed like "--seed 42" with the --seed option.
+
+For other options, please refer to the help with --help (there seems to be no documentation for the meaning of the parameters, so you have to look at the source).
+
+A caption file is generated with the extension .caption by default.
+
+![Folder where caption is generated](https://user-images.githubusercontent.com/52813779/208908845-48a9d36c-f6ee-4dae-af71-9ab462d1459e.png)
+
+For example, with captions like:
+
+![captions and images](https://user-images.githubusercontent.com/52813779/208908947-af936957-5d73-4339-b6c8-945a52857373.png)
+
+## Tagged by DeepDanbooru
+If you do not want to tag the danbooru tag itself, please proceed to "Preprocessing of caption and tag information".
+
+Tagging is done with DeepDanbooru or WD14Tagger. WD14Tagger seems to be more accurate. If you want to tag with WD14Tagger, skip to the next chapter.
+
+### Environmental arrangement
+Clone DeepDanbooru https://github.com/KichangKim/DeepDanbooru into your working folder, or download the zip and extract it. I unzipped it.
+Also, download deepdanbooru-v3-20211112-sgd-e28.zip from Assets of "DeepDanbooru Pretrained Model v3-20211112-sgd-e28" on the DeepDanbooru Releases page https://github.com/KichangKim/DeepDanbooru/releases and extract it to the DeepDanbooru folder.
+
+Download from below. Click to open Assets and download from there.
+
+![DeepDanbooru download page](https://user-images.githubusercontent.com/52813779/208909417-10e597df-7085-41ee-bd06-3e856a1339df.png)
+
+Make a directory structure like this
+
+![DeepDanbooru directory structure](https://user-images.githubusercontent.com/52813779/208909486-38935d8b-8dc6-43f1-84d3-fef99bc471aa.png)
+
+Install the necessary libraries for the Diffusers environment. Go to the DeepDanbooru folder and install it (I think it's actually just adding tensorflow-io).
+
+```
+pip install -r requirements.txt
+```
+
+Next, install DeepDanbooru itself.
+
+```
+pip install .
+```
+
+This completes the preparation of the environment for tagging.
+
+### Implementing tagging
+Go to DeepDanbooru's folder and run deepdanbooru to tag.
+
+```
+deepdanbooru evaluate <teacher data folder> --project-path deepdanbooru-v3-20211112-sgd-e28 --allow-folder --save-txt
+```
+
+If you put the training data in the parent folder train_data, it will be as follows.
+
+```
+deepdanbooru evaluate ../train_data --project-path deepdanbooru-v3-20211112-sgd-e28 --allow-folder --save-txt
+```
+
+A tag file is created in the same directory as the teacher data image with the same file name and extension .txt. It is slow because it is processed one by one.
+
+If there are multiple teacher data folders, execute for each folder.
+
+It is generated as follows.
+
+![DeepDanbooru generated files](https://user-images.githubusercontent.com/52813779/208909855-d21b9c98-f2d3-4283-8238-5b0e5aad6691.png)
+
+A tag is attached like this (great amount of information...).
+
+![Deep Danbooru tag and image](https://user-images.githubusercontent.com/52813779/208909908-a7920174-266e-48d5-aaef-940aba709519.png)
+
+## Tagging with WD14Tagger
+This procedure uses WD14Tagger instead of DeepDanbooru.
+
+Use the tagger used in Mr. Automatic1111's WebUI. I referred to the information on this github page (https://github.com/toriato/stable-diffusion-webui-wd14-tagger#mrsmilingwolfs-model-aka-waifu-diffusion-14-tagger).
+
+The modules required for the initial environment maintenance have already been installed. Weights are automatically downloaded from Hugging Face.
+
+### Implementing tagging
+Run the script to do the tagging.
+```
+python tag_images_by_wd14_tagger.py --batch_size <batch size> <teacher data folder>
+```
+
+If you put the training data in the parent folder train_data, it will be as follows.
+```
+python tag_images_by_wd14_tagger.py --batch_size 4 ..\train_data
+```
+
+The model file will be automatically downloaded to the wd14_tagger_model folder on first launch (folder can be changed in options). It will be as follows.
+
+![downloaded file](https://user-images.githubusercontent.com/52813779/208910447-f7eb0582-90d6-49d3-a666-2b508c7d1842.png)
+
+A tag file is created in the same directory as the teacher data image with the same file name and extension .txt.
+
+![generated tag file](https://user-images.githubusercontent.com/52813779/208910534-ea514373-1185-4b7d-9ae3-61eb50bc294e.png)
+
+![tags and images](https://user-images.githubusercontent.com/52813779/208910599-29070c15-7639-474f-b3e4-06bd5a3df29e.png)
+
+With the thresh option, you can specify the number of confidences of the determined tag to attach the tag. The default is 0.35, same as the WD14Tagger sample. Lower values give more tags, but less accuracy.
+Increase or decrease batch_size according to the VRAM capacity of the GPU. Bigger is faster (I think 12GB of VRAM can be a little more). You can change the tag file extension with the caption_extension option. Default is .txt.
+You can specify the folder where the model is saved with the model_dir option.
+Also, if you specify the force_download option, the model will be re-downloaded even if there is a save destination folder.
+
+If there are multiple teacher data folders, execute for each folder.
+
+## Preprocessing caption and tag information
+
+Combine captions and tags into a single file as metadata for easy processing from scripts.
+
+### Caption preprocessing
+
+To put captions into the metadata, run the following in your working folder (if you don't use captions for learning, you don't need to run this) (it's actually a single line, and so on).
+
+```
+python merge_captions_to_metadata.py <teacher data folder>
+--in_json <metadata file name to read>
+     <metadata file name>
+```
+
+The metadata file name is an arbitrary name.
+If the training data is train_data, there is no metadata file to read, and the metadata file is meta_cap.json, it will be as follows.
+
+```
+python merge_captions_to_metadata.py train_data meta_cap.json
+```
+
+You can specify the caption extension with the caption_extension option.
+
+If there are multiple teacher data folders, please specify the full_path argument (metadata will have full path information). Then run it for each folder.
+
+```
+python merge_captions_to_metadata.py --full_path
+     train_data1 meta_cap1.json
+python merge_captions_to_metadata.py --full_path --in_json meta_cap1.json
+     train_data2 meta_cap2.json
+```
+
+If in_json is omitted, if there is a write destination metadata file, it will be read from there and overwritten there.
+
+__*It is safe to rewrite the in_json option and the write destination each time and write to a separate metadata file. __
+
+### Tag preprocessing
+
+Similarly, tags are also collected in metadata (no need to do this if tags are not used for learning).
+```
+python merge_dd_tags_to_metadata.py <teacher data folder>
+     --in_json <metadata file name to load>
+     <metadata file name to write>
+```
+
+With the same directory structure as above, when reading meta_cap.json and writing to meta_cap_dd.json, it will be as follows.
+```
+python merge_dd_tags_to_metadata.py train_data --in_json meta_cap.json meta_cap_dd.json
+```
+
+If you have multiple teacher data folders, please specify the full_path argument. Then run it for each folder.
+
+```
+python merge_dd_tags_to_metadata.py --full_path --in_json meta_cap2.json
+     train_data1 meta_cap_dd1.json
+python merge_dd_tags_to_metadata.py --full_path --in_json meta_cap_dd1.json
+     train_data2 meta_cap_dd2.json
+```
+
+If in_json is omitted, if there is a write destination metadata file, it will be read from there and overwritten there.
+
+__*It is safe to rewrite the in_json option and the write destination each time and write to a separate metadata file. __
+
+### Cleaning captions and tags
+Up to this point, captions and DeepDanbooru tags have been put together in the metadata file. However, captions with automatic captioning are subtle due to spelling variations (*), and tags include underscores and ratings (in the case of DeepDanbooru), so the editor's replacement function etc. You should use it to clean your captions and tags.
+
+*For example, when learning a girl in an anime picture, there are variations in captions such as girl/girls/woman/women. Also, it may be more appropriate to simply use "girl" for things like "anime girl".
+
+A script for cleaning is provided, so please edit the contents of the script according to the situation and use it.
+
+(It is no longer necessary to specify the teacher data folder. All data in the metadata will be cleaned.)
+
+```
+python clean_captions_and_tags.py <metadata file name to read> <metadata file name to write>
+```
+
+Please note that --in_json is not included. For example:
+
+```
+python clean_captions_and_tags.py meta_cap_dd.json meta_clean.json
+```
+
+Preprocessing of captions and tags is now complete.
+
+## Get latents in advance
+
+In order to speed up the learning, we acquire the latent representation of the image in advance and save it to disk. At the same time, bucketing (classifying the training data according to the aspect ratio) is performed.
+
+In your working folder, type:
+```
+python prepare_buckets_latents.py <teacher data folder>
+     <metadata file name to read> <metadata file name to write>
+     <model name or checkpoint for fine tuning>
+     --batch_size <batch size>
+     --max_resolution <resolution width, height>
+     --mixed_precision <precision>
+```
+
+If the model is model.ckpt, batch size 4, training resolution is 512\*512, precision is no (float32), read metadata from meta_clean.json and write to meta_lat.json:
+
+```
+python prepare_buckets_latents.py
+     train_data meta_clean.json meta_lat.json model.ckpt
+     --batch_size 4 --max_resolution 512,512 --mixed_precision no
+```
+
+Latents are saved in numpy npz format in the teacher data folder.
+
+Specify the --v2 option when loading a Stable Diffusion 2.0 model (--v_parameterization is not required).
+
+You can specify the minimum resolution size with the --min_bucket_reso option and the maximum size with the --max_bucket_reso option. The defaults are 256 and 1024 respectively. For example, specifying a minimum size of 384 will not use resolutions such as 256\*1024 or 320\*768.
+If you increase the resolution to something like 768\*768, you should specify something like 1280 for the maximum size.
+
+If you specify the --flip_aug option, it will perform horizontal flip augmentation (data augmentation). You can artificially double the amount of data, but if you specify it when the data is not left-right symmetrical (for example, character appearance, hairstyle, etc.), learning will not go well.
+(This is a simple implementation that acquires the latents for the flipped image and saves the \*\_flip.npz file. No options are required for fline_tune.py. If there is a file with \_flip, Randomly load a file without
+
+The batch size may be increased a little more even with 12GB of VRAM.
+The resolution is a number divisible by 64, and is specified by "width, height". The resolution is directly linked to the memory size during fine tuning. 512,512 seems to be the limit with VRAM 12GB (*). 16GB may be raised to 512,704 or 512,768. Even with 256, 256, etc., it seems to be difficult with 8GB of VRAM (because parameters and optimizers require a certain amount of memory regardless of resolution).
+
+*There was also a report that learning batch size 1 worked with 12GB VRAM and 640,640.
+
+The result of bucketing is displayed as follows.
+
+![bucketing result](https://user-images.githubusercontent.com/52813779/208911419-71c00fbb-2ce6-49d5-89b5-b78d7715e441.png)
+
+If you have multiple teacher data folders, please specify the full_path argument. Then run it for each folder.
+```
+python prepare_buckets_latents.py --full_path
+     train_data1 meta_clean.json meta_lat1.json model.ckpt
+     --batch_size 4 --max_resolution 512,512 --mixed_precision no
+
+python prepare_buckets_latents.py --full_path
+     train_data2 meta_lat1.json meta_lat2.json model.ckpt
+     --batch_size 4 --max_resolution 512,512 --mixed_precision no
+
+```
+It is possible to make the read source and write destination the same, but separate is safer.
+
+__*It is safe to rewrite the argument each time and write it to a separate metadata file. __
+
+
+## Run training
+For example: Below are the settings for saving memory.
+```
+accelerate launch --num_cpu_threads_per_process 8 fine_tune.py
+     --pretrained_model_name_or_path=model.ckpt
+     --in_json meta_lat.json
+     --train_data_dir=train_data
+     --output_dir=fine_tuned
+     --shuffle_caption
+     --train_batch_size=1 --learning_rate=5e-6 --max_train_steps=10000
+     --use_8bit_adam --xformers --gradient_checkpointing
+     --mixed_precision=bf16
+     --save_every_n_epochs=4
+```
+
+It seems to be good to specify the number of CPU cores for num_cpu_threads_per_process of accelerate.
+
+Specify the model to be trained in pretrained_model_name_or_path (Stable Diffusion checkpoint or Diffusers model). Stable Diffusion checkpoint supports .ckpt and .safetensors (automatically determined by extension).
+
+Specifies the metadata file when caching latent to in_json.
+
+Specify the training data folder for train_data_dir and the output destination folder for the trained model for output_dir.
+
+If shuffle_caption is specified, captions and tags are shuffled and learned in units separated by commas (this is the method used in Waifu Diffusion v1.3).
+(You can keep some of the leading tokens fixed without shuffling. See keep_tokens for other options.)
+
+Specify the batch size in train_batch_size. Specify 1 or 2 for VRAM 12GB. The number that can be specified also changes depending on the resolution.
+The actual amount of data used for training is "batch size x number of steps". When increasing the batch size, the number of steps can be decreased accordingly.
+
+Specify the learning rate in learning_rate. For example Waifu Diffusion v1.3 seems to be 5e-6.
+Specify the number of steps in max_train_steps.
+
+Specify use_8bit_adam to use the 8-bit Adam Optimizer. It saves memory and speeds up, but accuracy may decrease.
+
+Specifying xformers replaces CrossAttention to save memory and speed up.
+* As of 11/9, xformers will cause an error in float32 learning, so please use bf16/fp16 or use memory-saving CrossAttention with mem_eff_attn instead (speed is inferior to xformers).
+
+Enable intermediate saving of gradients in gradient_checkpointing. It's slower, but uses less memory.
+
+Specifies whether to use mixed precision with mixed_precision. Specifying "fp16" or "bf16" saves memory, but accuracy is inferior.
+"fp16" and "bf16" use almost the same amount of memory, and it is said that bf16 has better learning results (I didn't feel much difference in the range I tried).
+If "no" is specified, it will not be used (it will be float32).
+
+* It seems that an error will occur when reading checkpoints learned with bf16 with Mr. AUTOMATIC1111's Web UI. This seems to be because the data type bfloat16 causes an error in the Web UI model safety checker. Save in fp16 or float32 format with the save_precision option. Or it seems to be good to store it in safetytensors format.
+
+Specifying save_every_n_epochs will save the model being trained every time that many epochs have passed.
+
+### Supports Stable Diffusion 2.0
+Specify the --v2 option when using Hugging Face's stable-diffusion-2-base, and specify both --v2 and --v_parameterization options when using stable-diffusion-2 or 768-v-ema.ckpt please.
+
+### Increase accuracy and speed when memory is available
+First, removing gradient_checkpointing will speed it up. However, the batch size that can be set is reduced, so please set while looking at the balance between accuracy and speed.
+
+Increasing the batch size increases speed and accuracy. Increase the speed while checking the speed per data within the range where the memory is sufficient (the speed may actually decrease when the memory is at the limit).
+
+### Change CLIP output used
+Specifying 2 for the clip_skip option uses the output of the next-to-last layer. If 1 or option is omitted, the last layer is used.
+The learned model should be able to be inferred by Automatic1111's web UI.
+
+*SD2.0 uses the second layer from the back by default, so please do not specify it when learning SD2.0.
+
+If the model being trained was originally trained to use the second layer, 2 is a good value.
+
+If you were using the last layer instead, the entire model would have been trained on that assumption. Therefore, if you train again using the second layer, you may need a certain number of teacher data and longer learning to obtain the desired learning result.
+
+### Extending Token Length
+You can learn by extending the token length by specifying 150 or 225 for max_token_length.
+The learned model should be able to be inferred by Automatic1111's web UI.
+
+As with clip_skip, learning with a length different from the learning state of the model may require a certain amount of teacher data and a longer learning time.
+
+### Save learning log
+Specify the log save destination folder in the logging_dir option. Logs in TensorBoard format are saved.
+
+For example, if you specify --logging_dir=logs, a logs folder will be created in your working folder, and logs will be saved in the date/time folder.
+Also, if you specify the --log_prefix option, the specified string will be added before the date and time. Use "--logging_dir=logs --log_prefix=fine_tune_style1" for identification.
+
+To check the log with TensorBoard, open another command prompt and enter the following in the working folder (I think tensorboard is installed when Diffusers is installed, but if it is not installed, pip install Please put it in tensorboard).
+```
+tensorboard --logdir=logs
+```
+
+### Learning Hypernetworks
+It will be explained in another article.
+
+### Learning with fp16 gradient (experimental feature)
+The full_fp16 option will change the gradient from normal float32 to float16 (fp16) and learn (it seems to be full fp16 learning instead of mixed precision). As a result, it seems that the SD1.x 512*512 size can be learned with a VRAM usage of less than 8GB, and the SD2.x 512*512 size can be learned with a VRAM usage of less than 12GB.
+
+Specify fp16 in advance in accelerate config and optionally set mixed_precision="fp16" (does not work with bf16).
+
+To minimize memory usage, use the xformers, use_8bit_adam, gradient_checkpointing options and set train_batch_size to 1.
+(If you can afford it, increasing the train_batch_size step by step should improve the accuracy a little.)
+
+It is realized by patching the PyTorch source (confirmed with PyTorch 1.12.1 and 1.13.0). The accuracy will drop considerably, and the probability of learning failure on the way will also increase. The setting of the learning rate and the number of steps seems to be severe. Please be aware of them and use them at your own risk.
+
+### Other Options
+
+#### keep_tokens
+If a number is specified, the specified number of tokens (comma-separated strings) from the beginning of the caption are fixed without being shuffled.
+
+If there are both captions and tags, the prompts during learning will be concatenated like "caption, tag 1, tag 2...", so if you set "--keep_tokens=1", the caption will always be at the beginning during learning. will come.
+
+#### dataset_repeats
+If the number of data sets is extremely small, the epoch will end soon (it will take some time at the epoch break), so please specify a numerical value and multiply the data by some to make the epoch longer.
+
+#### train_text_encoder
+Text Encoder is also a learning target. Slightly increased memory usage.
+
+In normal fine tuning, the Text Encoder is not targeted for training (probably because U-Net is trained to follow the output of the Text Encoder), but if the number of training data is small, the Text Encoder is trained like DreamBooth. also seems to be valid.
+
+#### save_precision
+The data format when saving checkpoints can be specified from float, fp16, and bf16 (if not specified, it is the same as the data format during learning). It saves disk space, but the model produces different results. Also, if you specify float or fp16, you should be able to read it on Mr. 1111's Web UI.
+
+*For VAE, the data format of the original checkpoint will remain, so the model size may not be reduced to a little over 2GB even with fp16.
+
+#### save_model_as
+Specify the save format of the model. Specify one of ckpt, safetensors, diffusers, diffusers_safetensors.
+
+When reading Stable Diffusion format (ckpt or safetensors) and saving in Diffusers format, missing information is supplemented by dropping v1.5 or v2.1 information from Hugging Face.
+
+#### use_safetensors
+This option saves checkpoints in safetyensors format. The save format will be the default (same format as loaded).
+
+#### save_state and resume
+The save_state option saves the learning state of the optimizer, etc. in addition to the checkpoint in the folder when saving midway and at the final save. This avoids a decrease in accuracy when learning is resumed after being interrupted (since the optimizer optimizes while having a state, if the state is reset, the optimization must be performed again from the initial state. not). Note that the number of steps is not saved due to Accelerate specifications.
+
+When starting the script, you can resume by specifying the folder where the state is saved with the resume option.
+
+Please note that the learning state will be about 5 GB per save, so please be careful of the disk capacity.
+
+#### gradient_accumulation_steps
+Updates the gradient in batches for the specified number of steps. Has a similar effect to increasing the batch size, but consumes slightly more memory.
+
+*The Accelerate specification does not support multiple learning models, so if you set Text Encoder as the learning target and specify a value of 2 or more for this option, an error may occur.
+
+#### lr_scheduler / lr_warmup_steps
+You can choose the learning rate scheduler from linear, cosine, cosine_with_restarts, polynomial, constant, constant_with_warmup with the lr_scheduler option. Default is constant.
+
+With lr_warmup_steps, you can specify the number of steps to warm up the scheduler (gradually changing the learning rate). Please do your own research for details.
+
+#### diffusers_xformers
+Uses Diffusers' xformers feature rather than the script's own xformers replacement feature. Hypernetwork learning is no longer possible.
\ No newline at end of file
diff --git a/finetune/blip/blip.py b/finetune/blip/blip.py
new file mode 100644
index 0000000000000000000000000000000000000000..7851fb08b21d15c93aab2a1d109f5018423b4e6b
--- /dev/null
+++ b/finetune/blip/blip.py
@@ -0,0 +1,240 @@
+'''
+ * Copyright (c) 2022, salesforce.com, inc.
+ * All rights reserved.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ * By Junnan Li
+'''
+import warnings
+warnings.filterwarnings("ignore")
+
+# from models.vit import VisionTransformer, interpolate_pos_embed
+# from models.med import BertConfig, BertModel, BertLMHeadModel
+from blip.vit import VisionTransformer, interpolate_pos_embed
+from blip.med import BertConfig, BertModel, BertLMHeadModel
+from transformers import BertTokenizer
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+
+import os
+from urllib.parse import urlparse
+from timm.models.hub import download_cached_file
+
+class BLIP_Base(nn.Module):
+    def __init__(self,                 
+                 med_config = 'configs/med_config.json',  
+                 image_size = 224,
+                 vit = 'base',
+                 vit_grad_ckpt = False,
+                 vit_ckpt_layer = 0,                 
+                 ):
+        """
+        Args:
+            med_config (str): path for the mixture of encoder-decoder model's configuration file
+            image_size (int): input image size
+            vit (str): model size of vision transformer
+        """               
+        super().__init__()
+        
+        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer)
+        self.tokenizer = init_tokenizer()   
+        med_config = BertConfig.from_json_file(med_config)
+        med_config.encoder_width = vision_width
+        self.text_encoder = BertModel(config=med_config, add_pooling_layer=False)  
+
+        
+    def forward(self, image, caption, mode):
+        
+        assert mode in ['image', 'text', 'multimodal'], "mode parameter must be image, text, or multimodal"
+        text = self.tokenizer(caption, return_tensors="pt").to(image.device) 
+        
+        if mode=='image':    
+            # return image features
+            image_embeds = self.visual_encoder(image)             
+            return image_embeds
+        
+        elif mode=='text':
+            # return text features
+            text_output = self.text_encoder(text.input_ids, attention_mask = text.attention_mask,                      
+                                            return_dict = True, mode = 'text')  
+            return text_output.last_hidden_state
+        
+        elif mode=='multimodal':
+            # return multimodel features
+            image_embeds = self.visual_encoder(image)    
+            image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)      
+            
+            text.input_ids[:,0] = self.tokenizer.enc_token_id
+            output = self.text_encoder(text.input_ids,
+                                       attention_mask = text.attention_mask,
+                                       encoder_hidden_states = image_embeds,
+                                       encoder_attention_mask = image_atts,      
+                                       return_dict = True,
+                                      )              
+            return output.last_hidden_state
+        
+        
+        
+class BLIP_Decoder(nn.Module):
+    def __init__(self,                 
+                 med_config = 'configs/med_config.json',  
+                 image_size = 384,
+                 vit = 'base',
+                 vit_grad_ckpt = False,
+                 vit_ckpt_layer = 0,
+                 prompt = 'a picture of ',
+                 ):
+        """
+        Args:
+            med_config (str): path for the mixture of encoder-decoder model's configuration file
+            image_size (int): input image size
+            vit (str): model size of vision transformer
+        """            
+        super().__init__()
+        
+        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer)
+        self.tokenizer = init_tokenizer()   
+        med_config = BertConfig.from_json_file(med_config)
+        med_config.encoder_width = vision_width
+        self.text_decoder = BertLMHeadModel(config=med_config)    
+        
+        self.prompt = prompt
+        self.prompt_length = len(self.tokenizer(self.prompt).input_ids)-1
+
+        
+    def forward(self, image, caption):
+        
+        image_embeds = self.visual_encoder(image) 
+        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)
+        
+        text = self.tokenizer(caption, padding='longest', truncation=True, max_length=40, return_tensors="pt").to(image.device) 
+        
+        text.input_ids[:,0] = self.tokenizer.bos_token_id
+        
+        decoder_targets = text.input_ids.masked_fill(text.input_ids == self.tokenizer.pad_token_id, -100)         
+        decoder_targets[:,:self.prompt_length] = -100
+     
+        decoder_output = self.text_decoder(text.input_ids, 
+                                           attention_mask = text.attention_mask, 
+                                           encoder_hidden_states = image_embeds,
+                                           encoder_attention_mask = image_atts,                  
+                                           labels = decoder_targets,
+                                           return_dict = True,   
+                                          )   
+        loss_lm = decoder_output.loss
+        
+        return loss_lm
+        
+    def generate(self, image, sample=False, num_beams=3, max_length=30, min_length=10, top_p=0.9, repetition_penalty=1.0):
+        image_embeds = self.visual_encoder(image)
+
+        if not sample:
+            image_embeds = image_embeds.repeat_interleave(num_beams,dim=0)
+            
+        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)
+        model_kwargs = {"encoder_hidden_states": image_embeds, "encoder_attention_mask":image_atts}
+        
+        prompt = [self.prompt] * image.size(0)
+        input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids.to(image.device) 
+        input_ids[:,0] = self.tokenizer.bos_token_id
+        input_ids = input_ids[:, :-1] 
+
+        if sample:
+            #nucleus sampling
+            outputs = self.text_decoder.generate(input_ids=input_ids,
+                                                  max_length=max_length,
+                                                  min_length=min_length,
+                                                  do_sample=True,
+                                                  top_p=top_p,
+                                                  num_return_sequences=1,
+                                                  eos_token_id=self.tokenizer.sep_token_id,
+                                                  pad_token_id=self.tokenizer.pad_token_id, 
+                                                  repetition_penalty=1.1,                                            
+                                                  **model_kwargs)
+        else:
+            #beam search
+            outputs = self.text_decoder.generate(input_ids=input_ids,
+                                                  max_length=max_length,
+                                                  min_length=min_length,
+                                                  num_beams=num_beams,
+                                                  eos_token_id=self.tokenizer.sep_token_id,
+                                                  pad_token_id=self.tokenizer.pad_token_id,     
+                                                  repetition_penalty=repetition_penalty,
+                                                  **model_kwargs)            
+            
+        captions = []    
+        for output in outputs:
+            caption = self.tokenizer.decode(output, skip_special_tokens=True)    
+            captions.append(caption[len(self.prompt):])
+        return captions
+    
+
+def blip_decoder(pretrained='',**kwargs):
+    model = BLIP_Decoder(**kwargs)
+    if pretrained:
+        model,msg = load_checkpoint(model,pretrained)
+        assert(len(msg.missing_keys)==0)
+    return model    
+    
+def blip_feature_extractor(pretrained='',**kwargs):
+    model = BLIP_Base(**kwargs)
+    if pretrained:
+        model,msg = load_checkpoint(model,pretrained)
+        assert(len(msg.missing_keys)==0)
+    return model        
+
+def init_tokenizer():
+    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+    tokenizer.add_special_tokens({'bos_token':'[DEC]'})
+    tokenizer.add_special_tokens({'additional_special_tokens':['[ENC]']})       
+    tokenizer.enc_token_id = tokenizer.additional_special_tokens_ids[0]  
+    return tokenizer
+
+
+def create_vit(vit, image_size, use_grad_checkpointing=False, ckpt_layer=0, drop_path_rate=0):
+        
+    assert vit in ['base', 'large'], "vit parameter must be base or large"
+    if vit=='base':
+        vision_width = 768
+        visual_encoder = VisionTransformer(img_size=image_size, patch_size=16, embed_dim=vision_width, depth=12, 
+                                           num_heads=12, use_grad_checkpointing=use_grad_checkpointing, ckpt_layer=ckpt_layer,
+                                           drop_path_rate=0 or drop_path_rate
+                                          )   
+    elif vit=='large':
+        vision_width = 1024
+        visual_encoder = VisionTransformer(img_size=image_size, patch_size=16, embed_dim=vision_width, depth=24, 
+                                           num_heads=16, use_grad_checkpointing=use_grad_checkpointing, ckpt_layer=ckpt_layer,
+                                           drop_path_rate=0.1 or drop_path_rate
+                                          )   
+    return visual_encoder, vision_width
+
+def is_url(url_or_filename):
+    parsed = urlparse(url_or_filename)
+    return parsed.scheme in ("http", "https")
+
+def load_checkpoint(model,url_or_filename):
+    if is_url(url_or_filename):
+        cached_file = download_cached_file(url_or_filename, check_hash=False, progress=True)
+        checkpoint = torch.load(cached_file, map_location='cpu') 
+    elif os.path.isfile(url_or_filename):        
+        checkpoint = torch.load(url_or_filename, map_location='cpu') 
+    else:
+        raise RuntimeError('checkpoint url or path is invalid')
+        
+    state_dict = checkpoint['model']
+    
+    state_dict['visual_encoder.pos_embed'] = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) 
+    if 'visual_encoder_m.pos_embed' in model.state_dict().keys():
+        state_dict['visual_encoder_m.pos_embed'] = interpolate_pos_embed(state_dict['visual_encoder_m.pos_embed'],
+                                                                         model.visual_encoder_m)    
+    for key in model.state_dict().keys():
+        if key in state_dict.keys():
+            if state_dict[key].shape!=model.state_dict()[key].shape:
+                del state_dict[key]
+    
+    msg = model.load_state_dict(state_dict,strict=False)
+    print('load checkpoint from %s'%url_or_filename)  
+    return model,msg
+    
diff --git a/finetune/blip/med.py b/finetune/blip/med.py
new file mode 100644
index 0000000000000000000000000000000000000000..7b00a35450b736180a805d4f4664b4fb95aeba01
--- /dev/null
+++ b/finetune/blip/med.py
@@ -0,0 +1,955 @@
+'''
+ * Copyright (c) 2022, salesforce.com, inc.
+ * All rights reserved.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ * By Junnan Li
+ * Based on huggingface code base
+ * https://github.com/huggingface/transformers/blob/v4.15.0/src/transformers/models/bert
+'''
+
+import math
+import os
+import warnings
+from dataclasses import dataclass
+from typing import Optional, Tuple
+
+import torch
+from torch import Tensor, device, dtype, nn
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import CrossEntropyLoss
+import torch.nn.functional as F
+
+from transformers.activations import ACT2FN
+from transformers.file_utils import (
+    ModelOutput,
+)
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPastAndCrossAttentions,
+    BaseModelOutputWithPoolingAndCrossAttentions,
+    CausalLMOutputWithCrossAttentions,
+    MaskedLMOutput,
+    MultipleChoiceModelOutput,
+    NextSentencePredictorOutput,
+    QuestionAnsweringModelOutput,
+    SequenceClassifierOutput,
+    TokenClassifierOutput,
+)
+from transformers.modeling_utils import (
+    PreTrainedModel,
+    apply_chunking_to_forward,
+    find_pruneable_heads_and_indices,
+    prune_linear_layer,
+)
+from transformers.utils import logging
+from transformers.models.bert.configuration_bert import BertConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+class BertEmbeddings(nn.Module):
+    """Construct the embeddings from word and position embeddings."""
+
+    def __init__(self, config):
+        super().__init__()
+        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
+        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
+
+        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
+        # any TensorFlow checkpoint file
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
+        self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)))
+        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
+        
+        self.config = config
+
+    def forward(
+        self, input_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0
+    ):
+        if input_ids is not None:
+            input_shape = input_ids.size()
+        else:
+            input_shape = inputs_embeds.size()[:-1]
+
+        seq_length = input_shape[1]
+
+        if position_ids is None:
+            position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]
+
+        if inputs_embeds is None:
+            inputs_embeds = self.word_embeddings(input_ids)
+
+        embeddings = inputs_embeds
+
+        if self.position_embedding_type == "absolute":
+            position_embeddings = self.position_embeddings(position_ids)
+            embeddings += position_embeddings
+        embeddings = self.LayerNorm(embeddings)
+        embeddings = self.dropout(embeddings)
+        return embeddings
+
+
+class BertSelfAttention(nn.Module):
+    def __init__(self, config, is_cross_attention):
+        super().__init__()
+        self.config = config
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
+            raise ValueError(
+                "The hidden size (%d) is not a multiple of the number of attention "
+                "heads (%d)" % (config.hidden_size, config.num_attention_heads)
+            )
+        
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        if is_cross_attention:
+            self.key = nn.Linear(config.encoder_width, self.all_head_size)
+            self.value = nn.Linear(config.encoder_width, self.all_head_size)
+        else:
+            self.key = nn.Linear(config.hidden_size, self.all_head_size)
+            self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
+        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
+            self.max_position_embeddings = config.max_position_embeddings
+            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)
+        self.save_attention = False   
+            
+    def save_attn_gradients(self, attn_gradients):
+        self.attn_gradients = attn_gradients
+        
+    def get_attn_gradients(self):
+        return self.attn_gradients
+    
+    def save_attention_map(self, attention_map):
+        self.attention_map = attention_map
+        
+    def get_attention_map(self):
+        return self.attention_map
+    
+    def transpose_for_scores(self, x):
+        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
+        x = x.view(*new_x_shape)
+        return x.permute(0, 2, 1, 3)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+    ):
+        mixed_query_layer = self.query(hidden_states)
+
+        # If this is instantiated as a cross-attention module, the keys
+        # and values come from an encoder; the attention mask needs to be
+        # such that the encoder's padding tokens are not attended to.
+        is_cross_attention = encoder_hidden_states is not None
+
+        if is_cross_attention:
+            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
+            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
+            attention_mask = encoder_attention_mask
+        elif past_key_value is not None:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+            key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
+            value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
+        else:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+
+        past_key_value = (key_layer, value_layer)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+
+        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
+            seq_length = hidden_states.size()[1]
+            position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1)
+            position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1)
+            distance = position_ids_l - position_ids_r
+            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
+            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility
+
+            if self.position_embedding_type == "relative_key":
+                relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
+                attention_scores = attention_scores + relative_position_scores
+            elif self.position_embedding_type == "relative_key_query":
+                relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
+                relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
+                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
+
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.Softmax(dim=-1)(attention_scores)
+        
+        if is_cross_attention and self.save_attention:
+            self.save_attention_map(attention_probs)
+            attention_probs.register_hook(self.save_attn_gradients)         
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs_dropped = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs_dropped = attention_probs_dropped * head_mask
+
+        context_layer = torch.matmul(attention_probs_dropped, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(*new_context_layer_shape)
+
+        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
+
+        outputs = outputs + (past_key_value,)
+        return outputs
+
+
+class BertSelfOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states, input_tensor):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class BertAttention(nn.Module):
+    def __init__(self, config, is_cross_attention=False):
+        super().__init__()
+        self.self = BertSelfAttention(config, is_cross_attention)
+        self.output = BertSelfOutput(config)
+        self.pruned_heads = set()
+
+    def prune_heads(self, heads):
+        if len(heads) == 0:
+            return
+        heads, index = find_pruneable_heads_and_indices(
+            heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
+        )
+
+        # Prune linear layers
+        self.self.query = prune_linear_layer(self.self.query, index)
+        self.self.key = prune_linear_layer(self.self.key, index)
+        self.self.value = prune_linear_layer(self.self.value, index)
+        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
+
+        # Update hyper params and store pruned heads
+        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
+        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
+        self.pruned_heads = self.pruned_heads.union(heads)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+    ):
+        self_outputs = self.self(
+            hidden_states,
+            attention_mask,
+            head_mask,
+            encoder_hidden_states,
+            encoder_attention_mask,
+            past_key_value,
+            output_attentions,
+        )
+        attention_output = self.output(self_outputs[0], hidden_states)
+        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
+        return outputs
+
+
+class BertIntermediate(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
+        if isinstance(config.hidden_act, str):
+            self.intermediate_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.intermediate_act_fn = config.hidden_act
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.intermediate_act_fn(hidden_states)
+        return hidden_states
+
+
+class BertOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states, input_tensor):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class BertLayer(nn.Module):
+    def __init__(self, config, layer_num):
+        super().__init__()
+        self.config = config
+        self.chunk_size_feed_forward = config.chunk_size_feed_forward
+        self.seq_len_dim = 1
+        self.attention = BertAttention(config)      
+        self.layer_num = layer_num          
+        if self.config.add_cross_attention:
+            self.crossattention = BertAttention(config, is_cross_attention=self.config.add_cross_attention)
+        self.intermediate = BertIntermediate(config)
+        self.output = BertOutput(config)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+        mode=None,
+    ):
+        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
+        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
+        self_attention_outputs = self.attention(
+            hidden_states,
+            attention_mask,
+            head_mask,
+            output_attentions=output_attentions,
+            past_key_value=self_attn_past_key_value,
+        )
+        attention_output = self_attention_outputs[0]
+
+        outputs = self_attention_outputs[1:-1]
+        present_key_value = self_attention_outputs[-1]
+
+        if mode=='multimodal':
+            assert encoder_hidden_states is not None, "encoder_hidden_states must be given for cross-attention layers"
+
+            cross_attention_outputs = self.crossattention(
+                attention_output,
+                attention_mask,
+                head_mask,
+                encoder_hidden_states,
+                encoder_attention_mask,
+                output_attentions=output_attentions,
+            )
+            attention_output = cross_attention_outputs[0]
+            outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights                               
+        layer_output = apply_chunking_to_forward(
+            self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
+        )
+        outputs = (layer_output,) + outputs
+
+        outputs = outputs + (present_key_value,)
+
+        return outputs
+
+    def feed_forward_chunk(self, attention_output):
+        intermediate_output = self.intermediate(attention_output)
+        layer_output = self.output(intermediate_output, attention_output)
+        return layer_output
+
+
+class BertEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.layer = nn.ModuleList([BertLayer(config,i) for i in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=False,
+        output_hidden_states=False,
+        return_dict=True,
+        mode='multimodal',
+    ):
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attentions = () if output_attentions else None
+        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
+
+        next_decoder_cache = () if use_cache else None
+               
+        for i in range(self.config.num_hidden_layers):
+            layer_module = self.layer[i]
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            layer_head_mask = head_mask[i] if head_mask is not None else None
+            past_key_value = past_key_values[i] if past_key_values is not None else None
+
+            if self.gradient_checkpointing and self.training:
+
+                if use_cache:
+                    logger.warn(
+                        "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                    )
+                    use_cache = False
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        return module(*inputs, past_key_value, output_attentions)
+
+                    return custom_forward
+
+                layer_outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(layer_module),
+                    hidden_states,
+                    attention_mask,
+                    layer_head_mask,
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    mode=mode,
+                )
+            else:
+                layer_outputs = layer_module(
+                    hidden_states,
+                    attention_mask,
+                    layer_head_mask,
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    past_key_value,
+                    output_attentions,
+                    mode=mode,
+                )
+
+            hidden_states = layer_outputs[0]
+            if use_cache:
+                next_decoder_cache += (layer_outputs[-1],)
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [
+                    hidden_states,
+                    next_decoder_cache,
+                    all_hidden_states,
+                    all_self_attentions,
+                    all_cross_attentions,
+                ]
+                if v is not None
+            )
+        return BaseModelOutputWithPastAndCrossAttentions(
+            last_hidden_state=hidden_states,
+            past_key_values=next_decoder_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+            cross_attentions=all_cross_attentions,
+        )
+
+
+class BertPooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = nn.Tanh()
+
+    def forward(self, hidden_states):
+        # We "pool" the model by simply taking the hidden state corresponding
+        # to the first token.
+        first_token_tensor = hidden_states[:, 0]
+        pooled_output = self.dense(first_token_tensor)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output
+
+
+class BertPredictionHeadTransform(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        if isinstance(config.hidden_act, str):
+            self.transform_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.transform_act_fn = config.hidden_act
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.transform_act_fn(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states)
+        return hidden_states
+
+
+class BertLMPredictionHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.transform = BertPredictionHeadTransform(config)
+
+        # The output weights are the same as the input embeddings, but there is
+        # an output-only bias for each token.
+        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
+
+        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
+        self.decoder.bias = self.bias
+
+    def forward(self, hidden_states):
+        hidden_states = self.transform(hidden_states)
+        hidden_states = self.decoder(hidden_states)
+        return hidden_states
+
+
+class BertOnlyMLMHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.predictions = BertLMPredictionHead(config)
+
+    def forward(self, sequence_output):
+        prediction_scores = self.predictions(sequence_output)
+        return prediction_scores
+
+
+class BertPreTrainedModel(PreTrainedModel):
+    """
+    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+    models.
+    """
+
+    config_class = BertConfig
+    base_model_prefix = "bert"
+    _keys_to_ignore_on_load_missing = [r"position_ids"]
+
+    def _init_weights(self, module):
+        """ Initialize the weights """
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            # Slightly different from the TF version which uses truncated_normal for initialization
+            # cf https://github.com/pytorch/pytorch/pull/5617
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+        if isinstance(module, nn.Linear) and module.bias is not None:
+            module.bias.data.zero_()
+
+
+class BertModel(BertPreTrainedModel):
+    """
+    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
+    cross-attention is added between the self-attention layers, following the architecture described in `Attention is
+    all you need <https://arxiv.org/abs/1706.03762>`__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
+    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
+    argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an
+    input to the forward pass.
+    """
+
+    def __init__(self, config, add_pooling_layer=True):
+        super().__init__(config)
+        self.config = config
+
+        self.embeddings = BertEmbeddings(config)
+        
+        self.encoder = BertEncoder(config)
+
+        self.pooler = BertPooler(config) if add_pooling_layer else None
+
+        self.init_weights()
+ 
+
+    def get_input_embeddings(self):
+        return self.embeddings.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.embeddings.word_embeddings = value
+
+    def _prune_heads(self, heads_to_prune):
+        """
+        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
+        class PreTrainedModel
+        """
+        for layer, heads in heads_to_prune.items():
+            self.encoder.layer[layer].attention.prune_heads(heads)
+
+    
+    def get_extended_attention_mask(self, attention_mask: Tensor, input_shape: Tuple[int], device: device, is_decoder: bool) -> Tensor:
+        """
+        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
+
+        Arguments:
+            attention_mask (:obj:`torch.Tensor`):
+                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
+            input_shape (:obj:`Tuple[int]`):
+                The shape of the input to the model.
+            device: (:obj:`torch.device`):
+                The device of the input to the model.
+
+        Returns:
+            :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`.
+        """
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        if attention_mask.dim() == 3:
+            extended_attention_mask = attention_mask[:, None, :, :]
+        elif attention_mask.dim() == 2:
+            # Provided a padding mask of dimensions [batch_size, seq_length]
+            # - if the model is a decoder, apply a causal mask in addition to the padding mask
+            # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
+            if is_decoder:
+                batch_size, seq_length = input_shape
+
+                seq_ids = torch.arange(seq_length, device=device)
+                causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
+                # in case past_key_values are used we need to add a prefix ones mask to the causal mask
+                # causal and attention masks must have same type with pytorch version < 1.3
+                causal_mask = causal_mask.to(attention_mask.dtype)
+   
+                if causal_mask.shape[1] < attention_mask.shape[1]:
+                    prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1]
+                    causal_mask = torch.cat(
+                        [
+                            torch.ones((batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype),
+                            causal_mask,
+                        ],
+                        axis=-1,
+                    )                     
+
+                extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
+            else:
+                extended_attention_mask = attention_mask[:, None, None, :]
+        else:
+            raise ValueError(
+                "Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(
+                    input_shape, attention_mask.shape
+                )
+            )
+
+        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+        # masked positions, this operation will create a tensor which is 0.0 for
+        # positions we want to attend and -10000.0 for masked positions.
+        # Since we are adding it to the raw scores before the softmax, this is
+        # effectively the same as removing these entirely.
+        extended_attention_mask = extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+        return extended_attention_mask
+    
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        encoder_embeds=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        is_decoder=False,
+        mode='multimodal',
+    ):
+        r"""
+        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
+            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
+            the model is configured as a decoder.
+        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
+            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
+            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
+            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
+            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
+            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
+            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
+        use_cache (:obj:`bool`, `optional`):
+            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
+            decoding (see :obj:`past_key_values`).
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if is_decoder:
+            use_cache = use_cache if use_cache is not None else self.config.use_cache
+        else:
+            use_cache = False
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            batch_size, seq_length = input_shape
+            device = input_ids.device
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+            batch_size, seq_length = input_shape
+            device = inputs_embeds.device
+        elif encoder_embeds is not None:    
+            input_shape = encoder_embeds.size()[:-1]
+            batch_size, seq_length = input_shape 
+            device = encoder_embeds.device
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds or encoder_embeds")
+
+        # past_key_values_length
+        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0
+
+        if attention_mask is None:
+            attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length)), device=device)
+            
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, 
+                                                                                 device, is_decoder)
+
+        # If a 2D or 3D attention mask is provided for the cross-attention
+        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
+        if encoder_hidden_states is not None:
+            if type(encoder_hidden_states) == list:
+                encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size()
+            else:
+                encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
+            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
+            
+            if type(encoder_attention_mask) == list:
+                encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask]
+            elif encoder_attention_mask is None:
+                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
+                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+            else:    
+                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+        else:
+            encoder_extended_attention_mask = None
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+        
+        if encoder_embeds is None:
+            embedding_output = self.embeddings(
+                input_ids=input_ids,
+                position_ids=position_ids,
+                inputs_embeds=inputs_embeds,
+                past_key_values_length=past_key_values_length,
+            )
+        else:
+            embedding_output = encoder_embeds
+            
+        encoder_outputs = self.encoder(
+            embedding_output,
+            attention_mask=extended_attention_mask,
+            head_mask=head_mask,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_extended_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            mode=mode,
+        )
+        sequence_output = encoder_outputs[0]
+        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
+
+        if not return_dict:
+            return (sequence_output, pooled_output) + encoder_outputs[1:]
+
+        return BaseModelOutputWithPoolingAndCrossAttentions(
+            last_hidden_state=sequence_output,
+            pooler_output=pooled_output,
+            past_key_values=encoder_outputs.past_key_values,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+            cross_attentions=encoder_outputs.cross_attentions,
+        )
+
+
+
+class BertLMHeadModel(BertPreTrainedModel):
+
+    _keys_to_ignore_on_load_unexpected = [r"pooler"]
+    _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"]
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.bert = BertModel(config, add_pooling_layer=False)
+        self.cls = BertOnlyMLMHead(config)
+
+        self.init_weights()
+
+    def get_output_embeddings(self):
+        return self.cls.predictions.decoder
+
+    def set_output_embeddings(self, new_embeddings):
+        self.cls.predictions.decoder = new_embeddings
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        labels=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        return_logits=False,            
+        is_decoder=True,
+        reduction='mean',
+        mode='multimodal', 
+    ):
+        r"""
+        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
+            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
+            the model is configured as a decoder.
+        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
+            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
+            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
+            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
+            ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are
+            ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]``
+        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
+            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
+            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
+            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
+            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
+        use_cache (:obj:`bool`, `optional`):
+            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
+            decoding (see :obj:`past_key_values`).
+        Returns:
+        Example::
+            >>> from transformers import BertTokenizer, BertLMHeadModel, BertConfig
+            >>> import torch
+            >>> tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
+            >>> config = BertConfig.from_pretrained("bert-base-cased")
+            >>> model = BertLMHeadModel.from_pretrained('bert-base-cased', config=config)
+            >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
+            >>> outputs = model(**inputs)
+            >>> prediction_logits = outputs.logits
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        if labels is not None:
+            use_cache = False
+
+        outputs = self.bert(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            is_decoder=is_decoder,
+            mode=mode,
+        )
+        
+        sequence_output = outputs[0]
+        prediction_scores = self.cls(sequence_output)
+        
+        if return_logits:
+            return prediction_scores[:, :-1, :].contiguous()  
+
+        lm_loss = None
+        if labels is not None:
+            # we are doing next-token prediction; shift prediction scores and input ids by one
+            shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous()
+            labels = labels[:, 1:].contiguous()
+            loss_fct = CrossEntropyLoss(reduction=reduction, label_smoothing=0.1) 
+            lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
+            if reduction=='none':
+                lm_loss = lm_loss.view(prediction_scores.size(0),-1).sum(1)               
+
+        if not return_dict:
+            output = (prediction_scores,) + outputs[2:]
+            return ((lm_loss,) + output) if lm_loss is not None else output
+
+        return CausalLMOutputWithCrossAttentions(
+            loss=lm_loss,
+            logits=prediction_scores,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            cross_attentions=outputs.cross_attentions,
+        )
+
+    def prepare_inputs_for_generation(self, input_ids, past=None, attention_mask=None, **model_kwargs):
+        input_shape = input_ids.shape
+        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
+        if attention_mask is None:
+            attention_mask = input_ids.new_ones(input_shape)
+
+        # cut decoder_input_ids if past is used
+        if past is not None:
+            input_ids = input_ids[:, -1:]
+
+        return {
+            "input_ids": input_ids, 
+            "attention_mask": attention_mask, 
+            "past_key_values": past,
+            "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None),
+            "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None),
+            "is_decoder": True,
+        }
+
+    def _reorder_cache(self, past, beam_idx):
+        reordered_past = ()
+        for layer_past in past:
+            reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),)
+        return reordered_past
diff --git a/finetune/blip/med_config.json b/finetune/blip/med_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..dc12b99cf539b751d442b4ca7785c9f6a4f8306e
--- /dev/null
+++ b/finetune/blip/med_config.json
@@ -0,0 +1,22 @@
+{
+    "architectures": [
+      "BertModel"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-12,
+    "max_position_embeddings": 512,
+    "model_type": "bert",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "pad_token_id": 0,
+    "type_vocab_size": 2,
+    "vocab_size": 30524,
+    "encoder_width": 768,
+    "add_cross_attention": true   
+  }
+  
\ No newline at end of file
diff --git a/finetune/blip/vit.py b/finetune/blip/vit.py
new file mode 100644
index 0000000000000000000000000000000000000000..cec3d8e08ed4451d65392feb2e9f4848d1ef3899
--- /dev/null
+++ b/finetune/blip/vit.py
@@ -0,0 +1,305 @@
+'''
+ * Copyright (c) 2022, salesforce.com, inc.
+ * All rights reserved.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ * By Junnan Li
+ * Based on timm code base
+ * https://github.com/rwightman/pytorch-image-models/tree/master/timm
+'''
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from functools import partial
+
+from timm.models.vision_transformer import _cfg, PatchEmbed
+from timm.models.registry import register_model
+from timm.models.layers import trunc_normal_, DropPath
+from timm.models.helpers import named_apply, adapt_input_conv
+
+from fairscale.nn.checkpoint.checkpoint_activations import checkpoint_wrapper
+
+class Mlp(nn.Module):
+    """ MLP as used in Vision Transformer, MLP-Mixer and related networks
+    """
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Attention(nn.Module):
+    def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights
+        self.scale = qk_scale or head_dim ** -0.5
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+        self.attn_gradients = None
+        self.attention_map = None
+        
+    def save_attn_gradients(self, attn_gradients):
+        self.attn_gradients = attn_gradients
+        
+    def get_attn_gradients(self):
+        return self.attn_gradients
+    
+    def save_attention_map(self, attention_map):
+        self.attention_map = attention_map
+        
+    def get_attention_map(self):
+        return self.attention_map
+    
+    def forward(self, x, register_hook=False):
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]   # make torchscript happy (cannot use tensor as tuple)
+
+        attn = (q @ k.transpose(-2, -1)) * self.scale
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+                
+        if register_hook:
+            self.save_attention_map(attn)
+            attn.register_hook(self.save_attn_gradients)        
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class Block(nn.Module):
+
+    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
+                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, use_grad_checkpointing=False):
+        super().__init__()
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(
+            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+
+        if use_grad_checkpointing:
+            self.attn = checkpoint_wrapper(self.attn)
+            self.mlp = checkpoint_wrapper(self.mlp)
+
+    def forward(self, x, register_hook=False):
+        x = x + self.drop_path(self.attn(self.norm1(x), register_hook=register_hook))
+        x = x + self.drop_path(self.mlp(self.norm2(x)))
+        return x
+
+    
+class VisionTransformer(nn.Module):
+    """ Vision Transformer
+    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`  -
+        https://arxiv.org/abs/2010.11929
+    """
+    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
+                 num_heads=12, mlp_ratio=4., qkv_bias=True, qk_scale=None, representation_size=None,
+                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None, 
+                 use_grad_checkpointing=False, ckpt_layer=0):
+        """
+        Args:
+            img_size (int, tuple): input image size
+            patch_size (int, tuple): patch size
+            in_chans (int): number of input channels
+            num_classes (int): number of classes for classification head
+            embed_dim (int): embedding dimension
+            depth (int): depth of transformer
+            num_heads (int): number of attention heads
+            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
+            qkv_bias (bool): enable bias for qkv if True
+            qk_scale (float): override default qk scale of head_dim ** -0.5 if set
+            representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set
+            drop_rate (float): dropout rate
+            attn_drop_rate (float): attention dropout rate
+            drop_path_rate (float): stochastic depth rate
+            norm_layer: (nn.Module): normalization layer
+        """
+        super().__init__()
+        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
+        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
+
+        self.patch_embed = PatchEmbed(
+            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
+
+        num_patches = self.patch_embed.num_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
+        self.blocks = nn.ModuleList([
+            Block(
+                dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
+                drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer,
+                use_grad_checkpointing=(use_grad_checkpointing and i>=depth-ckpt_layer)
+            )
+            for i in range(depth)])
+        self.norm = norm_layer(embed_dim)
+
+        trunc_normal_(self.pos_embed, std=.02)
+        trunc_normal_(self.cls_token, std=.02)
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    @torch.jit.ignore
+    def no_weight_decay(self):
+        return {'pos_embed', 'cls_token'}
+
+    def forward(self, x, register_blk=-1):
+        B = x.shape[0]
+        x = self.patch_embed(x)
+
+        cls_tokens = self.cls_token.expand(B, -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
+        x = torch.cat((cls_tokens, x), dim=1)
+  
+        x = x + self.pos_embed[:,:x.size(1),:]
+        x = self.pos_drop(x)
+
+        for i,blk in enumerate(self.blocks):
+            x = blk(x, register_blk==i)
+        x = self.norm(x)
+        
+        return x
+
+    @torch.jit.ignore()
+    def load_pretrained(self, checkpoint_path, prefix=''):
+        _load_weights(self, checkpoint_path, prefix)
+        
+
+@torch.no_grad()
+def _load_weights(model: VisionTransformer, checkpoint_path: str, prefix: str = ''):
+    """ Load weights from .npz checkpoints for official Google Brain Flax implementation
+    """
+    import numpy as np
+
+    def _n2p(w, t=True):
+        if w.ndim == 4 and w.shape[0] == w.shape[1] == w.shape[2] == 1:
+            w = w.flatten()
+        if t:
+            if w.ndim == 4:
+                w = w.transpose([3, 2, 0, 1])
+            elif w.ndim == 3:
+                w = w.transpose([2, 0, 1])
+            elif w.ndim == 2:
+                w = w.transpose([1, 0])
+        return torch.from_numpy(w)
+
+    w = np.load(checkpoint_path)
+    if not prefix and 'opt/target/embedding/kernel' in w:
+        prefix = 'opt/target/'
+
+    if hasattr(model.patch_embed, 'backbone'):
+        # hybrid
+        backbone = model.patch_embed.backbone
+        stem_only = not hasattr(backbone, 'stem')
+        stem = backbone if stem_only else backbone.stem
+        stem.conv.weight.copy_(adapt_input_conv(stem.conv.weight.shape[1], _n2p(w[f'{prefix}conv_root/kernel'])))
+        stem.norm.weight.copy_(_n2p(w[f'{prefix}gn_root/scale']))
+        stem.norm.bias.copy_(_n2p(w[f'{prefix}gn_root/bias']))
+        if not stem_only:
+            for i, stage in enumerate(backbone.stages):
+                for j, block in enumerate(stage.blocks):
+                    bp = f'{prefix}block{i + 1}/unit{j + 1}/'
+                    for r in range(3):
+                        getattr(block, f'conv{r + 1}').weight.copy_(_n2p(w[f'{bp}conv{r + 1}/kernel']))
+                        getattr(block, f'norm{r + 1}').weight.copy_(_n2p(w[f'{bp}gn{r + 1}/scale']))
+                        getattr(block, f'norm{r + 1}').bias.copy_(_n2p(w[f'{bp}gn{r + 1}/bias']))
+                    if block.downsample is not None:
+                        block.downsample.conv.weight.copy_(_n2p(w[f'{bp}conv_proj/kernel']))
+                        block.downsample.norm.weight.copy_(_n2p(w[f'{bp}gn_proj/scale']))
+                        block.downsample.norm.bias.copy_(_n2p(w[f'{bp}gn_proj/bias']))
+        embed_conv_w = _n2p(w[f'{prefix}embedding/kernel'])
+    else:
+        embed_conv_w = adapt_input_conv(
+            model.patch_embed.proj.weight.shape[1], _n2p(w[f'{prefix}embedding/kernel']))
+    model.patch_embed.proj.weight.copy_(embed_conv_w)
+    model.patch_embed.proj.bias.copy_(_n2p(w[f'{prefix}embedding/bias']))
+    model.cls_token.copy_(_n2p(w[f'{prefix}cls'], t=False))
+    pos_embed_w = _n2p(w[f'{prefix}Transformer/posembed_input/pos_embedding'], t=False)
+    if pos_embed_w.shape != model.pos_embed.shape:
+        pos_embed_w = resize_pos_embed(  # resize pos embedding when different size from pretrained weights
+            pos_embed_w, model.pos_embed, getattr(model, 'num_tokens', 1), model.patch_embed.grid_size)
+    model.pos_embed.copy_(pos_embed_w)
+    model.norm.weight.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/scale']))
+    model.norm.bias.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/bias']))
+#     if isinstance(model.head, nn.Linear) and model.head.bias.shape[0] == w[f'{prefix}head/bias'].shape[-1]:
+#         model.head.weight.copy_(_n2p(w[f'{prefix}head/kernel']))
+#         model.head.bias.copy_(_n2p(w[f'{prefix}head/bias']))
+#     if isinstance(getattr(model.pre_logits, 'fc', None), nn.Linear) and f'{prefix}pre_logits/bias' in w:
+#         model.pre_logits.fc.weight.copy_(_n2p(w[f'{prefix}pre_logits/kernel']))
+#         model.pre_logits.fc.bias.copy_(_n2p(w[f'{prefix}pre_logits/bias']))
+    for i, block in enumerate(model.blocks.children()):
+        block_prefix = f'{prefix}Transformer/encoderblock_{i}/'
+        mha_prefix = block_prefix + 'MultiHeadDotProductAttention_1/'
+        block.norm1.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/scale']))
+        block.norm1.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/bias']))
+        block.attn.qkv.weight.copy_(torch.cat([
+            _n2p(w[f'{mha_prefix}{n}/kernel'], t=False).flatten(1).T for n in ('query', 'key', 'value')]))
+        block.attn.qkv.bias.copy_(torch.cat([
+            _n2p(w[f'{mha_prefix}{n}/bias'], t=False).reshape(-1) for n in ('query', 'key', 'value')]))
+        block.attn.proj.weight.copy_(_n2p(w[f'{mha_prefix}out/kernel']).flatten(1))
+        block.attn.proj.bias.copy_(_n2p(w[f'{mha_prefix}out/bias']))
+        for r in range(2):
+            getattr(block.mlp, f'fc{r + 1}').weight.copy_(_n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/kernel']))
+            getattr(block.mlp, f'fc{r + 1}').bias.copy_(_n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/bias']))
+        block.norm2.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/scale']))
+        block.norm2.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/bias']))
+
+            
+def interpolate_pos_embed(pos_embed_checkpoint, visual_encoder):        
+    # interpolate position embedding
+    embedding_size = pos_embed_checkpoint.shape[-1]
+    num_patches = visual_encoder.patch_embed.num_patches
+    num_extra_tokens = visual_encoder.pos_embed.shape[-2] - num_patches
+    # height (== width) for the checkpoint position embedding
+    orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5)
+    # height (== width) for the new position embedding
+    new_size = int(num_patches ** 0.5)
+
+    if orig_size!=new_size:
+        # class_token and dist_token are kept unchanged
+        extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+        # only the position tokens are interpolated
+        pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+        pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
+        pos_tokens = torch.nn.functional.interpolate(
+            pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
+        pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
+        new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+        print('reshape position embedding from %d to %d'%(orig_size ** 2,new_size ** 2))
+        
+        return new_pos_embed    
+    else:
+        return pos_embed_checkpoint
\ No newline at end of file
diff --git a/finetune/clean_captions_and_tags.py b/finetune/clean_captions_and_tags.py
new file mode 100644
index 0000000000000000000000000000000000000000..68839ecccbbaf056204be1d6fb0d204e104091e6
--- /dev/null
+++ b/finetune/clean_captions_and_tags.py
@@ -0,0 +1,190 @@
+# このスクリプトのライセンスは、Apache License 2.0とします
+# (c) 2022 Kohya S. @kohya_ss
+
+import argparse
+import glob
+import os
+import json
+import re
+
+from tqdm import tqdm
+
+PATTERN_HAIR_LENGTH = re.compile(r', (long|short|medium) hair, ')
+PATTERN_HAIR_CUT = re.compile(r', (bob|hime) cut, ')
+PATTERN_HAIR = re.compile(r', ([\w\-]+) hair, ')
+PATTERN_WORD = re.compile(r', ([\w\-]+|hair ornament), ')
+
+# 複数人がいるとき、複数の髪色や目の色が定義されていれば削除する
+PATTERNS_REMOVE_IN_MULTI = [
+    PATTERN_HAIR_LENGTH,
+    PATTERN_HAIR_CUT,
+    re.compile(r', [\w\-]+ eyes, '),
+    re.compile(r', ([\w\-]+ sleeves|sleeveless), '),
+    # 複数の髪型定義がある場合は削除する
+    re.compile(
+        r', (ponytail|braid|ahoge|twintails|[\w\-]+ bun|single hair bun|single side bun|two side up|two tails|[\w\-]+ braid|sidelocks), '),
+]
+
+
+def clean_tags(image_key, tags):
+  # replace '_' to ' '
+  tags = tags.replace('^_^', '^@@@^')
+  tags = tags.replace('_', ' ')
+  tags = tags.replace('^@@@^', '^_^')
+
+  # remove rating: deepdanbooruのみ
+  tokens = tags.split(", rating")
+  if len(tokens) == 1:
+    # WD14 taggerのときはこちらになるのでメッセージは出さない
+    # print("no rating:")
+    # print(f"{image_key} {tags}")
+    pass
+  else:
+    if len(tokens) > 2:
+      print("multiple ratings:")
+      print(f"{image_key} {tags}")
+    tags = tokens[0]
+
+  tags = ", " + tags.replace(", ", ", , ") + ", "     # カンマ付きで検索をするための身も蓋もない対策
+  
+  # 複数の人物がいる場合は髪色等のタグを削除する
+  if 'girls' in tags or 'boys' in tags:
+    for pat in PATTERNS_REMOVE_IN_MULTI:
+      found = pat.findall(tags)
+      if len(found) > 1:                        # 二つ以上、タグがある
+        tags = pat.sub("", tags)
+
+    # 髪の特殊対応
+    srch_hair_len = PATTERN_HAIR_LENGTH.search(tags)   # 髪の長さタグは例外なので避けておく（全員が同じ髪の長さの場合）
+    if srch_hair_len:
+      org = srch_hair_len.group()
+      tags = PATTERN_HAIR_LENGTH.sub(", @@@, ", tags)
+
+    found = PATTERN_HAIR.findall(tags)
+    if len(found) > 1:
+      tags = PATTERN_HAIR.sub("", tags)
+
+    if srch_hair_len:
+      tags = tags.replace(", @@@, ", org)                   # 戻す
+
+  # white shirtとshirtみたいな重複タグの削除
+  found = PATTERN_WORD.findall(tags)
+  for word in found:
+    if re.search(f", ((\w+) )+{word}, ", tags):
+      tags = tags.replace(f", {word}, ", "")
+
+  tags = tags.replace(", , ", ", ")
+  assert tags.startswith(", ") and tags.endswith(", ")
+  tags = tags[2:-2]
+  return tags
+
+
+# 上から順に検索、置換される
+# ('置換元文字列', '置換後文字列')
+CAPTION_REPLACEMENTS = [
+    ('anime anime', 'anime'),
+    ('young ', ''),
+    ('anime girl', 'girl'),
+    ('cartoon female', 'girl'),
+    ('cartoon lady', 'girl'),
+    ('cartoon character', 'girl'),      # a or ~s
+    ('cartoon woman', 'girl'),
+    ('cartoon women', 'girls'),
+    ('cartoon girl', 'girl'),
+    ('anime female', 'girl'),
+    ('anime lady', 'girl'),
+    ('anime character', 'girl'),      # a or ~s
+    ('anime woman', 'girl'),
+    ('anime women', 'girls'),
+    ('lady', 'girl'),
+    ('female', 'girl'),
+    ('woman', 'girl'),
+    ('women', 'girls'),
+    ('people', 'girls'),
+    ('person', 'girl'),
+    ('a cartoon figure', 'a figure'),
+    ('a cartoon image', 'an image'),
+    ('a cartoon picture', 'a picture'),
+    ('an anime cartoon image', 'an image'),
+    ('a cartoon anime drawing', 'a drawing'),
+    ('a cartoon drawing', 'a drawing'),
+    ('girl girl', 'girl'),
+]
+
+
+def clean_caption(caption):
+  for rf, rt in CAPTION_REPLACEMENTS:
+    replaced = True
+    while replaced:
+      bef = caption
+      caption = caption.replace(rf, rt)
+      replaced = bef != caption
+  return caption
+
+
+def main(args):
+  if os.path.exists(args.in_json):
+    print(f"loading existing metadata: {args.in_json}")
+    with open(args.in_json, "rt", encoding='utf-8') as f:
+      metadata = json.load(f)
+  else:
+    print("no metadata / メタデータファイルがありません")
+    return
+
+  print("cleaning captions and tags.")
+  image_keys = list(metadata.keys())
+  for image_key in tqdm(image_keys):
+    tags = metadata[image_key].get('tags')
+    if tags is None:
+      print(f"image does not have tags / メタデータにタグがありません: {image_key}")
+    else:
+      org = tags
+      tags = clean_tags(image_key, tags)
+      metadata[image_key]['tags'] = tags
+      if args.debug and org != tags:
+        print("FROM: " + org)
+        print("TO:   " + tags)
+
+    caption = metadata[image_key].get('caption')
+    if caption is None:
+      print(f"image does not have caption / メタデータにキャプションがありません: {image_key}")
+    else:
+      org = caption
+      caption = clean_caption(caption)
+      metadata[image_key]['caption'] = caption
+      if args.debug and org != caption:
+        print("FROM: " + org)
+        print("TO:   " + caption)
+
+  # metadataを書き出して終わり
+  print(f"writing metadata: {args.out_json}")
+  with open(args.out_json, "wt", encoding='utf-8') as f:
+    json.dump(metadata, f, indent=2)
+  print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+  # parser.add_argument("train_data_dir", type=str, help="directory for train images / 学習画像データのディレクトリ")
+  parser.add_argument("in_json", type=str, help="metadata file to input / 読み込むメタデータファイル")
+  parser.add_argument("out_json", type=str, help="metadata file to output / メタデータファイル書き出し先")
+  parser.add_argument("--debug", action="store_true", help="debug mode")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args, unknown = parser.parse_known_args()
+  if len(unknown) == 1:
+    print("WARNING: train_data_dir argument is removed. This script will not work with three arguments in future. Please specify two arguments: in_json and out_json.")
+    print("All captions and tags in the metadata are processed.")
+    print("警告: train_data_dir引数は不要になりました。将来的には三つの引数を指定すると動かなくなる予定です。読み込み元のメタデータと書き出し先の二つの引数だけ指定してください。")
+    print("メタデータ内のすべてのキャプションとタグが処理されます。")
+    args.in_json = args.out_json
+    args.out_json = unknown[0]
+  elif len(unknown) > 0:
+    raise ValueError(f"error: unrecognized arguments: {unknown}")
+
+  main(args)
diff --git a/finetune/hypernetwork_nai.py b/finetune/hypernetwork_nai.py
new file mode 100644
index 0000000000000000000000000000000000000000..dcaaa714a08bb2cfc417d827e8bdd01c8c1ad367
--- /dev/null
+++ b/finetune/hypernetwork_nai.py
@@ -0,0 +1,96 @@
+# NAI compatible
+
+import torch
+
+
+class HypernetworkModule(torch.nn.Module):
+  def __init__(self, dim, multiplier=1.0):
+    super().__init__()
+
+    linear1 = torch.nn.Linear(dim, dim * 2)
+    linear2 = torch.nn.Linear(dim * 2, dim)
+    linear1.weight.data.normal_(mean=0.0, std=0.01)
+    linear1.bias.data.zero_()
+    linear2.weight.data.normal_(mean=0.0, std=0.01)
+    linear2.bias.data.zero_()
+    linears = [linear1, linear2]
+
+    self.linear = torch.nn.Sequential(*linears)
+    self.multiplier = multiplier
+
+  def forward(self, x):
+    return x + self.linear(x) * self.multiplier
+
+
+class Hypernetwork(torch.nn.Module):
+  enable_sizes = [320, 640, 768, 1280]
+  # return self.modules[Hypernetwork.enable_sizes.index(size)]
+
+  def __init__(self, multiplier=1.0) -> None:
+    super().__init__()
+    self.modules = []
+    for size in Hypernetwork.enable_sizes:
+      self.modules.append((HypernetworkModule(size, multiplier), HypernetworkModule(size, multiplier)))
+      self.register_module(f"{size}_0", self.modules[-1][0])
+      self.register_module(f"{size}_1", self.modules[-1][1])
+
+  def apply_to_stable_diffusion(self, text_encoder, vae, unet):
+    blocks = unet.input_blocks + [unet.middle_block] + unet.output_blocks
+    for block in blocks:
+      for subblk in block:
+        if 'SpatialTransformer' in str(type(subblk)):
+          for tf_block in subblk.transformer_blocks:
+            for attn in [tf_block.attn1, tf_block.attn2]:
+              size = attn.context_dim
+              if size in Hypernetwork.enable_sizes:
+                attn.hypernetwork = self
+              else:
+                attn.hypernetwork = None
+
+  def apply_to_diffusers(self, text_encoder, vae, unet):
+    blocks = unet.down_blocks + [unet.mid_block] + unet.up_blocks
+    for block in blocks:
+      if hasattr(block, 'attentions'):
+        for subblk in block.attentions:
+          if 'SpatialTransformer' in str(type(subblk)) or 'Transformer2DModel' in str(type(subblk)):      # 0.6.0 and 0.7~
+            for tf_block in subblk.transformer_blocks:
+              for attn in [tf_block.attn1, tf_block.attn2]:
+                size = attn.to_k.in_features
+                if size in Hypernetwork.enable_sizes:
+                  attn.hypernetwork = self
+                else:
+                  attn.hypernetwork = None
+    return True       # TODO error checking
+
+  def forward(self, x, context):
+    size = context.shape[-1]
+    assert size in Hypernetwork.enable_sizes
+    module = self.modules[Hypernetwork.enable_sizes.index(size)]
+    return module[0].forward(context), module[1].forward(context)
+
+  def load_from_state_dict(self, state_dict):
+    # old ver to new ver
+    changes = {
+        'linear1.bias': 'linear.0.bias',
+        'linear1.weight': 'linear.0.weight',
+        'linear2.bias': 'linear.1.bias',
+        'linear2.weight': 'linear.1.weight',
+    }
+    for key_from, key_to in changes.items():
+      if key_from in state_dict:
+        state_dict[key_to] = state_dict[key_from]
+        del state_dict[key_from]
+
+    for size, sd in state_dict.items():
+      if type(size) == int:
+        self.modules[Hypernetwork.enable_sizes.index(size)][0].load_state_dict(sd[0], strict=True)
+        self.modules[Hypernetwork.enable_sizes.index(size)][1].load_state_dict(sd[1], strict=True)
+    return True
+
+  def get_state_dict(self):
+    state_dict = {}
+    for i, size in enumerate(Hypernetwork.enable_sizes):
+      sd0 = self.modules[i][0].state_dict()
+      sd1 = self.modules[i][1].state_dict()
+      state_dict[size] = [sd0, sd1]
+    return state_dict
diff --git a/finetune/make_captions.py b/finetune/make_captions.py
new file mode 100644
index 0000000000000000000000000000000000000000..b20c41068fac3b101d444cbea994f691b93980eb
--- /dev/null
+++ b/finetune/make_captions.py
@@ -0,0 +1,200 @@
+import argparse
+import glob
+import os
+import json
+import random
+import sys
+
+from pathlib import Path
+from PIL import Image
+from tqdm import tqdm
+import numpy as np
+import torch
+from torchvision import transforms
+from torchvision.transforms.functional import InterpolationMode
+sys.path.append(os.path.dirname(__file__))
+from blip.blip import blip_decoder
+import library.train_util as train_util
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+IMAGE_SIZE = 384
+
+# 正方形でいいのか？　という気がするがソースがそうなので
+IMAGE_TRANSFORM = transforms.Compose(
+    [
+        transforms.Resize((IMAGE_SIZE, IMAGE_SIZE), interpolation=InterpolationMode.BICUBIC),
+        transforms.ToTensor(),
+        transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
+    ]
+)
+
+
+# 共通化したいが微妙に処理が異なる……
+class ImageLoadingTransformDataset(torch.utils.data.Dataset):
+    def __init__(self, image_paths):
+        self.images = image_paths
+
+    def __len__(self):
+        return len(self.images)
+
+    def __getitem__(self, idx):
+        img_path = self.images[idx]
+
+        try:
+            image = Image.open(img_path).convert("RGB")
+            # convert to tensor temporarily so dataloader will accept it
+            tensor = IMAGE_TRANSFORM(image)
+        except Exception as e:
+            print(f"Could not load image path / 画像を読み込めません: {img_path}, error: {e}")
+            return None
+
+        return (tensor, img_path)
+
+
+def collate_fn_remove_corrupted(batch):
+    """Collate function that allows to remove corrupted examples in the
+    dataloader. It expects that the dataloader returns 'None' when that occurs.
+    The 'None's in the batch are removed.
+    """
+    # Filter out all the Nones (corrupted examples)
+    batch = list(filter(lambda x: x is not None, batch))
+    return batch
+
+
+def main(args):
+    # fix the seed for reproducibility
+    seed = args.seed  # + utils.get_rank()
+    torch.manual_seed(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+
+    if not os.path.exists("blip"):
+        args.train_data_dir = os.path.abspath(args.train_data_dir)  # convert to absolute path
+
+        cwd = os.getcwd()
+        print("Current Working Directory is: ", cwd)
+        os.chdir("finetune")
+
+    print(f"load images from {args.train_data_dir}")
+    train_data_dir_path = Path(args.train_data_dir)
+    image_paths = train_util.glob_images_pathlib(train_data_dir_path, args.recursive)
+    print(f"found {len(image_paths)} images.")
+
+    print(f"loading BLIP caption: {args.caption_weights}")
+    model = blip_decoder(pretrained=args.caption_weights, image_size=IMAGE_SIZE, vit="large", med_config="./blip/med_config.json")
+    model.eval()
+    model = model.to(DEVICE)
+    print("BLIP loaded")
+
+    # captioningする
+    def run_batch(path_imgs):
+        imgs = torch.stack([im for _, im in path_imgs]).to(DEVICE)
+
+        with torch.no_grad():
+            if args.beam_search:
+                captions = model.generate(
+                    imgs, sample=False, num_beams=args.num_beams, max_length=args.max_length, min_length=args.min_length
+                )
+            else:
+                captions = model.generate(
+                    imgs, sample=True, top_p=args.top_p, max_length=args.max_length, min_length=args.min_length
+                )
+
+        for (image_path, _), caption in zip(path_imgs, captions):
+            with open(os.path.splitext(image_path)[0] + args.caption_extension, "wt", encoding="utf-8") as f:
+                f.write(caption + "\n")
+                if args.debug:
+                    print(image_path, caption)
+
+    # 読み込みの高速化のためにDataLoaderを使うオプション
+    if args.max_data_loader_n_workers is not None:
+        dataset = ImageLoadingTransformDataset(image_paths)
+        data = torch.utils.data.DataLoader(
+            dataset,
+            batch_size=args.batch_size,
+            shuffle=False,
+            num_workers=args.max_data_loader_n_workers,
+            collate_fn=collate_fn_remove_corrupted,
+            drop_last=False,
+        )
+    else:
+        data = [[(None, ip)] for ip in image_paths]
+
+    b_imgs = []
+    for data_entry in tqdm(data, smoothing=0.0):
+        for data in data_entry:
+            if data is None:
+                continue
+
+            img_tensor, image_path = data
+            if img_tensor is None:
+                try:
+                    raw_image = Image.open(image_path)
+                    if raw_image.mode != "RGB":
+                        raw_image = raw_image.convert("RGB")
+                    img_tensor = IMAGE_TRANSFORM(raw_image)
+                except Exception as e:
+                    print(f"Could not load image path / 画像を読み込めません: {image_path}, error: {e}")
+                    continue
+
+            b_imgs.append((image_path, img_tensor))
+            if len(b_imgs) >= args.batch_size:
+                run_batch(b_imgs)
+                b_imgs.clear()
+    if len(b_imgs) > 0:
+        run_batch(b_imgs)
+
+    print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("train_data_dir", type=str, help="directory for train images / 学習画像データのディレクトリ")
+    parser.add_argument(
+        "--caption_weights",
+        type=str,
+        default="https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_large_caption.pth",
+        help="BLIP caption weights (model_large_caption.pth) / BLIP captionの重みファイル(model_large_caption.pth)",
+    )
+    parser.add_argument(
+        "--caption_extention",
+        type=str,
+        default=None,
+        help="extension of caption file (for backward compatibility) / 出力されるキャプションファイルの拡張子（スペルミスしていたのを残してあります）",
+    )
+    parser.add_argument("--caption_extension", type=str, default=".caption", help="extension of caption file / 出力されるキャプションファイルの拡張子")
+    parser.add_argument(
+        "--beam_search",
+        action="store_true",
+        help="use beam search (default Nucleus sampling) / beam searchを使う（このオプション未指定時はNucleus sampling）",
+    )
+    parser.add_argument("--batch_size", type=int, default=1, help="batch size in inference / 推論時のバッチサイズ")
+    parser.add_argument(
+        "--max_data_loader_n_workers",
+        type=int,
+        default=None,
+        help="enable image reading by DataLoader with this number of workers (faster) / DataLoaderによる画像読み込みを有効にしてこのワーカー数を適用する（読み込みを高速化）",
+    )
+    parser.add_argument("--num_beams", type=int, default=1, help="num of beams in beam search /beam search時のビーム数（多いと精度が上がるが時間がかかる）")
+    parser.add_argument("--top_p", type=float, default=0.9, help="top_p in Nucleus sampling / Nucleus sampling時のtop_p")
+    parser.add_argument("--max_length", type=int, default=75, help="max length of caption / captionの最大長")
+    parser.add_argument("--min_length", type=int, default=5, help="min length of caption / captionの最小長")
+    parser.add_argument("--seed", default=42, type=int, help="seed for reproducibility / 再現性を確保するための乱数seed")
+    parser.add_argument("--debug", action="store_true", help="debug mode")
+    parser.add_argument("--recursive", action="store_true", help="search for images in subfolders recursively / サブフォルダを再帰的に検索する")
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+
+    # スペルミスしていたオプションを復元する
+    if args.caption_extention is not None:
+        args.caption_extension = args.caption_extention
+
+    main(args)
diff --git a/finetune/make_captions_by_git.py b/finetune/make_captions_by_git.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce6e6695517ef90887314fb148fffe5bacf79409
--- /dev/null
+++ b/finetune/make_captions_by_git.py
@@ -0,0 +1,172 @@
+import argparse
+import os
+import re
+
+from pathlib import Path
+from PIL import Image
+from tqdm import tqdm
+import torch
+from transformers import AutoProcessor, AutoModelForCausalLM
+from transformers.generation.utils import GenerationMixin
+
+import library.train_util as train_util
+
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+PATTERN_REPLACE = [
+    re.compile(r'(has|with|and) the (words?|letters?|name) (" ?[^"]*"|\w+)( ?(is )?(on|in) (the |her |their |him )?\w+)?'),
+    re.compile(r'(with a sign )?that says ?(" ?[^"]*"|\w+)( ?on it)?'),
+    re.compile(r"(with a sign )?that says ?(' ?(i'm)?[^']*'|\w+)( ?on it)?"),
+    re.compile(r"with the number \d+ on (it|\w+ \w+)"),
+    re.compile(r'with the words "'),
+    re.compile(r"word \w+ on it"),
+    re.compile(r"that says the word \w+ on it"),
+    re.compile("that says'the word \"( on it)?"),
+]
+
+# 誤検知しまくりの with the word xxxx を消す
+
+
+def remove_words(captions, debug):
+    removed_caps = []
+    for caption in captions:
+        cap = caption
+        for pat in PATTERN_REPLACE:
+            cap = pat.sub("", cap)
+        if debug and cap != caption:
+            print(caption)
+            print(cap)
+        removed_caps.append(cap)
+    return removed_caps
+
+
+def collate_fn_remove_corrupted(batch):
+    """Collate function that allows to remove corrupted examples in the
+    dataloader. It expects that the dataloader returns 'None' when that occurs.
+    The 'None's in the batch are removed.
+    """
+    # Filter out all the Nones (corrupted examples)
+    batch = list(filter(lambda x: x is not None, batch))
+    return batch
+
+
+def main(args):
+    # GITにバッチサイズが1より大きくても動くようにパッチを当てる: transformers 4.26.0用
+    org_prepare_input_ids_for_generation = GenerationMixin._prepare_input_ids_for_generation
+    curr_batch_size = [args.batch_size]  # ループの最後で件数がbatch_size未満になるので入れ替えられるように
+
+    # input_idsがバッチサイズと同じ件数である必要がある：バッチサイズはこの関数から参照できないので外から渡す
+    # ここより上で置き換えようとするとすごく大変
+    def _prepare_input_ids_for_generation_patch(self, bos_token_id, encoder_outputs):
+        input_ids = org_prepare_input_ids_for_generation(self, bos_token_id, encoder_outputs)
+        if input_ids.size()[0] != curr_batch_size[0]:
+            input_ids = input_ids.repeat(curr_batch_size[0], 1)
+        return input_ids
+
+    GenerationMixin._prepare_input_ids_for_generation = _prepare_input_ids_for_generation_patch
+
+    print(f"load images from {args.train_data_dir}")
+    train_data_dir_path = Path(args.train_data_dir)
+    image_paths = train_util.glob_images_pathlib(train_data_dir_path, args.recursive)
+    print(f"found {len(image_paths)} images.")
+
+    # できればcacheに依存せず明示的にダウンロードしたい
+    print(f"loading GIT: {args.model_id}")
+    git_processor = AutoProcessor.from_pretrained(args.model_id)
+    git_model = AutoModelForCausalLM.from_pretrained(args.model_id).to(DEVICE)
+    print("GIT loaded")
+
+    # captioningする
+    def run_batch(path_imgs):
+        imgs = [im for _, im in path_imgs]
+
+        curr_batch_size[0] = len(path_imgs)
+        inputs = git_processor(images=imgs, return_tensors="pt").to(DEVICE)  # 画像はpil形式
+        generated_ids = git_model.generate(pixel_values=inputs.pixel_values, max_length=args.max_length)
+        captions = git_processor.batch_decode(generated_ids, skip_special_tokens=True)
+
+        if args.remove_words:
+            captions = remove_words(captions, args.debug)
+
+        for (image_path, _), caption in zip(path_imgs, captions):
+            with open(os.path.splitext(image_path)[0] + args.caption_extension, "wt", encoding="utf-8") as f:
+                f.write(caption + "\n")
+                if args.debug:
+                    print(image_path, caption)
+
+    # 読み込みの高速化のためにDataLoaderを使うオプション
+    if args.max_data_loader_n_workers is not None:
+        dataset = train_util.ImageLoadingDataset(image_paths)
+        data = torch.utils.data.DataLoader(
+            dataset,
+            batch_size=args.batch_size,
+            shuffle=False,
+            num_workers=args.max_data_loader_n_workers,
+            collate_fn=collate_fn_remove_corrupted,
+            drop_last=False,
+        )
+    else:
+        data = [[(None, ip)] for ip in image_paths]
+
+    b_imgs = []
+    for data_entry in tqdm(data, smoothing=0.0):
+        for data in data_entry:
+            if data is None:
+                continue
+
+            image, image_path = data
+            if image is None:
+                try:
+                    image = Image.open(image_path)
+                    if image.mode != "RGB":
+                        image = image.convert("RGB")
+                except Exception as e:
+                    print(f"Could not load image path / 画像を読み込めません: {image_path}, error: {e}")
+                    continue
+
+            b_imgs.append((image_path, image))
+            if len(b_imgs) >= args.batch_size:
+                run_batch(b_imgs)
+                b_imgs.clear()
+
+    if len(b_imgs) > 0:
+        run_batch(b_imgs)
+
+    print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("train_data_dir", type=str, help="directory for train images / 学習画像データのディレクトリ")
+    parser.add_argument("--caption_extension", type=str, default=".caption", help="extension of caption file / 出力されるキャプションファイルの拡張子")
+    parser.add_argument(
+        "--model_id",
+        type=str,
+        default="microsoft/git-large-textcaps",
+        help="model id for GIT in Hugging Face / 使用するGITのHugging FaceのモデルID",
+    )
+    parser.add_argument("--batch_size", type=int, default=1, help="batch size in inference / 推論時のバッチサイズ")
+    parser.add_argument(
+        "--max_data_loader_n_workers",
+        type=int,
+        default=None,
+        help="enable image reading by DataLoader with this number of workers (faster) / DataLoaderによる画像読み込みを有効にしてこのワーカー数を適用する（読み込みを高速化）",
+    )
+    parser.add_argument("--max_length", type=int, default=50, help="max length of caption / captionの最大長")
+    parser.add_argument(
+        "--remove_words",
+        action="store_true",
+        help="remove like `with the words xxx` from caption / `with the words xxx`のような部分をキャプションから削除する",
+    )
+    parser.add_argument("--debug", action="store_true", help="debug mode")
+    parser.add_argument("--recursive", action="store_true", help="search for images in subfolders recursively / サブフォルダを再帰的に検索する")
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    main(args)
diff --git a/finetune/merge_captions_to_metadata.py b/finetune/merge_captions_to_metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..241f6f902867dfe8c8ba2cd8bed9f3553fd5b07f
--- /dev/null
+++ b/finetune/merge_captions_to_metadata.py
@@ -0,0 +1,76 @@
+import argparse
+import json
+from pathlib import Path
+from typing import List
+from tqdm import tqdm
+import library.train_util as train_util
+import os
+
+def main(args):
+  assert not args.recursive or (args.recursive and args.full_path), "recursive requires full_path / recursiveはfull_pathと同時に指定してください"
+
+  train_data_dir_path = Path(args.train_data_dir)
+  image_paths: List[Path] = train_util.glob_images_pathlib(train_data_dir_path, args.recursive)
+  print(f"found {len(image_paths)} images.")
+
+  if args.in_json is None and Path(args.out_json).is_file():
+    args.in_json = args.out_json
+
+  if args.in_json is not None:
+    print(f"loading existing metadata: {args.in_json}")
+    metadata = json.loads(Path(args.in_json).read_text(encoding='utf-8'))
+    print("captions for existing images will be overwritten / 既存の画像のキャプションは上書きされます")
+  else:
+    print("new metadata will be created / 新しいメタデータファイルが作成されます")
+    metadata = {}
+
+  print("merge caption texts to metadata json.")
+  for image_path in tqdm(image_paths):
+    caption_path = image_path.with_suffix(args.caption_extension)
+    caption = caption_path.read_text(encoding='utf-8').strip()
+
+    if not os.path.exists(caption_path):
+      caption_path = os.path.join(image_path, args.caption_extension)
+
+    image_key = str(image_path) if args.full_path else image_path.stem
+    if image_key not in metadata:
+      metadata[image_key] = {}
+
+    metadata[image_key]['caption'] = caption
+    if args.debug:
+      print(image_key, caption)
+
+  # metadataを書き出して終わり
+  print(f"writing metadata: {args.out_json}")
+  Path(args.out_json).write_text(json.dumps(metadata, indent=2), encoding='utf-8')
+  print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("train_data_dir", type=str, help="directory for train images / 学習画像データのディレクトリ")
+  parser.add_argument("out_json", type=str, help="metadata file to output / メタデータファイル書き出し先")
+  parser.add_argument("--in_json", type=str,
+                      help="metadata file to input (if omitted and out_json exists, existing out_json is read) / 読み込むメタデータファイル（省略時、out_jsonが存在すればそれを読み込む）")
+  parser.add_argument("--caption_extention", type=str, default=None,
+                      help="extension of caption file (for backward compatibility) / 読み込むキャプションファイルの拡張子（スペルミスしていたのを残してあります）")
+  parser.add_argument("--caption_extension", type=str, default=".caption", help="extension of caption file / 読み込むキャプションファイルの拡張子")
+  parser.add_argument("--full_path", action="store_true",
+                      help="use full path as image-key in metadata (supports multiple directories) / メタデータで画像キーをフルパスにする（複数の学習画像ディレクトリに対応）")
+  parser.add_argument("--recursive", action="store_true",
+                      help="recursively look for training tags in all child folders of train_data_dir / train_data_dirのすべての子フォルダにある学習タグを再帰的に探す")
+  parser.add_argument("--debug", action="store_true", help="debug mode")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+
+  # スペルミスしていたオプションを復元する
+  if args.caption_extention is not None:
+    args.caption_extension = args.caption_extention
+
+  main(args)
diff --git a/finetune/merge_dd_tags_to_metadata.py b/finetune/merge_dd_tags_to_metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..db1bff6da7a2227e2e04558de8c8f93e8523d2f9
--- /dev/null
+++ b/finetune/merge_dd_tags_to_metadata.py
@@ -0,0 +1,71 @@
+import argparse
+import json
+from pathlib import Path
+from typing import List
+from tqdm import tqdm
+import library.train_util as train_util
+import os
+
+def main(args):
+  assert not args.recursive or (args.recursive and args.full_path), "recursive requires full_path / recursiveはfull_pathと同時に指定してください"
+
+  train_data_dir_path = Path(args.train_data_dir)
+  image_paths: List[Path] = train_util.glob_images_pathlib(train_data_dir_path, args.recursive)
+  print(f"found {len(image_paths)} images.")
+
+  if args.in_json is None and Path(args.out_json).is_file():
+    args.in_json = args.out_json
+
+  if args.in_json is not None:
+    print(f"loading existing metadata: {args.in_json}")
+    metadata = json.loads(Path(args.in_json).read_text(encoding='utf-8'))
+    print("tags data for existing images will be overwritten / 既存の画像のタグは上書きされます")
+  else:
+    print("new metadata will be created / 新しいメタデータファイルが作成されます")
+    metadata = {}
+
+  print("merge tags to metadata json.")
+  for image_path in tqdm(image_paths):
+    tags_path = image_path.with_suffix(args.caption_extension)
+    tags = tags_path.read_text(encoding='utf-8').strip()
+
+    if not os.path.exists(tags_path):
+      tags_path = os.path.join(image_path, args.caption_extension)
+
+    image_key = str(image_path) if args.full_path else image_path.stem
+    if image_key not in metadata:
+      metadata[image_key] = {}
+
+    metadata[image_key]['tags'] = tags
+    if args.debug:
+      print(image_key, tags)
+
+  # metadataを書き出して終わり
+  print(f"writing metadata: {args.out_json}")
+  Path(args.out_json).write_text(json.dumps(metadata, indent=2), encoding='utf-8')
+
+  print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("train_data_dir", type=str, help="directory for train images / 学習画像データのディレクトリ")
+  parser.add_argument("out_json", type=str, help="metadata file to output / メタデータファイル書き出し先")
+  parser.add_argument("--in_json", type=str,
+                      help="metadata file to input (if omitted and out_json exists, existing out_json is read) / 読み込むメタデータファイル（省略時、out_jsonが存在すればそれを読み込む）")
+  parser.add_argument("--full_path", action="store_true",
+                      help="use full path as image-key in metadata (supports multiple directories) / メタデータで画像キーをフルパスにする（複数の学習画像ディレクトリに対応）")
+  parser.add_argument("--recursive", action="store_true",
+                      help="recursively look for training tags in all child folders of train_data_dir / train_data_dirのすべての子フォルダにある学習タグを再帰的に探す")
+  parser.add_argument("--caption_extension", type=str, default=".txt",
+                      help="extension of caption (tag) file / 読み込むキャプション（タグ）ファイルの拡張子")
+  parser.add_argument("--debug", action="store_true", help="debug mode, print tags")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+  main(args)
diff --git a/finetune/prepare_buckets_latents.py b/finetune/prepare_buckets_latents.py
new file mode 100644
index 0000000000000000000000000000000000000000..6bb1c32fe70fd7487b136d1fdc152f8157c55bcf
--- /dev/null
+++ b/finetune/prepare_buckets_latents.py
@@ -0,0 +1,343 @@
+import argparse
+import os
+import json
+
+from pathlib import Path
+from typing import List
+from tqdm import tqdm
+import numpy as np
+from PIL import Image
+import cv2
+import torch
+from torchvision import transforms
+
+import library.model_util as model_util
+import library.train_util as train_util
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+IMAGE_TRANSFORMS = transforms.Compose(
+    [
+        transforms.ToTensor(),
+        transforms.Normalize([0.5], [0.5]),
+    ]
+)
+
+
+def collate_fn_remove_corrupted(batch):
+    """Collate function that allows to remove corrupted examples in the
+    dataloader. It expects that the dataloader returns 'None' when that occurs.
+    The 'None's in the batch are removed.
+    """
+    # Filter out all the Nones (corrupted examples)
+    batch = list(filter(lambda x: x is not None, batch))
+    return batch
+
+
+def get_latents(vae, key_and_images, weight_dtype):
+    img_tensors = [IMAGE_TRANSFORMS(image) for _, image in key_and_images]
+    img_tensors = torch.stack(img_tensors)
+    img_tensors = img_tensors.to(DEVICE, weight_dtype)
+    with torch.no_grad():
+        latents = vae.encode(img_tensors).latent_dist.sample()
+        
+    # check NaN
+    for (key, _), latents1 in zip(key_and_images, latents):
+        if torch.isnan(latents1).any():
+            raise ValueError(f"NaN detected in latents of {key}")
+
+    return latents
+
+
+def get_npz_filename_wo_ext(data_dir, image_key, is_full_path, flip, recursive):
+    if is_full_path:
+        base_name = os.path.splitext(os.path.basename(image_key))[0]
+        relative_path = os.path.relpath(os.path.dirname(image_key), data_dir)
+    else:
+        base_name = image_key
+        relative_path = ""
+
+    if flip:
+        base_name += "_flip"
+
+    if recursive and relative_path:
+        return os.path.join(data_dir, relative_path, base_name)
+    else:
+        return os.path.join(data_dir, base_name)
+
+
+def main(args):
+    # assert args.bucket_reso_steps % 8 == 0, f"bucket_reso_steps must be divisible by 8 / bucket_reso_stepは8で割り切れる必要があります"
+    if args.bucket_reso_steps % 8 > 0:
+        print(f"resolution of buckets in training time is a multiple of 8 / 学習時の各bucketの解像度は8単位になります")
+
+    train_data_dir_path = Path(args.train_data_dir)
+    image_paths: List[str] = [str(p) for p in train_util.glob_images_pathlib(train_data_dir_path, args.recursive)]
+    print(f"found {len(image_paths)} images.")
+
+    if os.path.exists(args.in_json):
+        print(f"loading existing metadata: {args.in_json}")
+        with open(args.in_json, "rt", encoding="utf-8") as f:
+            metadata = json.load(f)
+    else:
+        print(f"no metadata / メタデータファイルがありません: {args.in_json}")
+        return
+
+    weight_dtype = torch.float32
+    if args.mixed_precision == "fp16":
+        weight_dtype = torch.float16
+    elif args.mixed_precision == "bf16":
+        weight_dtype = torch.bfloat16
+
+    vae = model_util.load_vae(args.model_name_or_path, weight_dtype)
+    vae.eval()
+    vae.to(DEVICE, dtype=weight_dtype)
+
+    # bucketのサイズを計算する
+    max_reso = tuple([int(t) for t in args.max_resolution.split(",")])
+    assert len(max_reso) == 2, f"illegal resolution (not 'width,height') / 画像サイズに誤りがあります。'幅,高さ'で指定してください: {args.max_resolution}"
+
+    bucket_manager = train_util.BucketManager(
+        args.bucket_no_upscale, max_reso, args.min_bucket_reso, args.max_bucket_reso, args.bucket_reso_steps
+    )
+    if not args.bucket_no_upscale:
+        bucket_manager.make_buckets()
+    else:
+        print(
+            "min_bucket_reso and max_bucket_reso are ignored if bucket_no_upscale is set, because bucket reso is defined by image size automatically / bucket_no_upscaleが指定された場合は、bucketの解像度は画像サイズから自動計算されるため、min_bucket_resoとmax_bucket_resoは無視されます"
+        )
+
+    # 画像をひとつずつ適切なbucketに割り当てながらlatentを計算する
+    img_ar_errors = []
+
+    def process_batch(is_last):
+        for bucket in bucket_manager.buckets:
+            if (is_last and len(bucket) > 0) or len(bucket) >= args.batch_size:
+                latents = get_latents(vae, [(key, img) for key, img, _, _ in bucket], weight_dtype)
+                assert (
+                    latents.shape[2] == bucket[0][1].shape[0] // 8 and latents.shape[3] == bucket[0][1].shape[1] // 8
+                ), f"latent shape {latents.shape}, {bucket[0][1].shape}"
+
+                for (image_key, _, original_size, crop_left_top), latent in zip(bucket, latents):
+                    npz_file_name = get_npz_filename_wo_ext(args.train_data_dir, image_key, args.full_path, False, args.recursive)
+                    train_util.save_latents_to_disk(npz_file_name, latent, original_size, crop_left_top)
+
+                # flip
+                if args.flip_aug:
+                    latents = get_latents(
+                        vae, [(key, img[:, ::-1].copy()) for key, img, _, _ in bucket], weight_dtype
+                    )  # copyがないとTensor変換できない
+
+                    for (image_key, _, original_size, crop_left_top), latent in zip(bucket, latents):
+                        npz_file_name = get_npz_filename_wo_ext(
+                            args.train_data_dir, image_key, args.full_path, True, args.recursive
+                        )
+                        train_util.save_latents_to_disk(npz_file_name, latent, original_size, crop_left_top)
+                else:
+                    # remove existing flipped npz
+                    for image_key, _ in bucket:
+                        npz_file_name = (
+                            get_npz_filename_wo_ext(args.train_data_dir, image_key, args.full_path, True, args.recursive) + ".npz"
+                        )
+                        if os.path.isfile(npz_file_name):
+                            print(f"remove existing flipped npz / 既存のflipされたnpzファイルを削除します: {npz_file_name}")
+                            os.remove(npz_file_name)
+
+                bucket.clear()
+
+    # 読み込みの高速化のためにDataLoaderを使うオプション
+    if args.max_data_loader_n_workers is not None:
+        dataset = train_util.ImageLoadingDataset(image_paths)
+        data = torch.utils.data.DataLoader(
+            dataset,
+            batch_size=1,
+            shuffle=False,
+            num_workers=args.max_data_loader_n_workers,
+            collate_fn=collate_fn_remove_corrupted,
+            drop_last=False,
+        )
+    else:
+        data = [[(None, ip)] for ip in image_paths]
+
+    bucket_counts = {}
+    for data_entry in tqdm(data, smoothing=0.0):
+        if data_entry[0] is None:
+            continue
+
+        img_tensor, image_path = data_entry[0]
+        if img_tensor is not None:
+            image = transforms.functional.to_pil_image(img_tensor)
+        else:
+            try:
+                image = Image.open(image_path)
+                if image.mode != "RGB":
+                    image = image.convert("RGB")
+            except Exception as e:
+                print(f"Could not load image path / 画像を読み込めません: {image_path}, error: {e}")
+                continue
+
+        image_key = image_path if args.full_path else os.path.splitext(os.path.basename(image_path))[0]
+        if image_key not in metadata:
+            metadata[image_key] = {}
+
+        # 本当はこのあとの部分もDataSetに持っていけば高速化できるがいろいろ大変
+
+        reso, resized_size, ar_error = bucket_manager.select_bucket(image.width, image.height)
+        img_ar_errors.append(abs(ar_error))
+        bucket_counts[reso] = bucket_counts.get(reso, 0) + 1
+
+        # メタデータに記録する解像度はlatent単位とするので、8単位で切り捨て
+        metadata[image_key]["train_resolution"] = (reso[0] - reso[0] % 8, reso[1] - reso[1] % 8)
+
+        if not args.bucket_no_upscale:
+            # upscaleを行わないときには、resize後のサイズは、bucketのサイズと、縦横どちらかが同じであることを確認する
+            assert (
+                resized_size[0] == reso[0] or resized_size[1] == reso[1]
+            ), f"internal error, resized size not match: {reso}, {resized_size}, {image.width}, {image.height}"
+            assert (
+                resized_size[0] >= reso[0] and resized_size[1] >= reso[1]
+            ), f"internal error, resized size too small: {reso}, {resized_size}, {image.width}, {image.height}"
+
+        assert (
+            resized_size[0] >= reso[0] and resized_size[1] >= reso[1]
+        ), f"internal error resized size is small: {resized_size}, {reso}"
+
+        # 既に存在するファイルがあればshape等を確認して同じならskipする
+        if args.skip_existing:
+            npz_files = [get_npz_filename_wo_ext(args.train_data_dir, image_key, args.full_path, False, args.recursive) + ".npz"]
+            if args.flip_aug:
+                npz_files.append(
+                    get_npz_filename_wo_ext(args.train_data_dir, image_key, args.full_path, True, args.recursive) + ".npz"
+                )
+
+            found = True
+            for npz_file in npz_files:
+                if not os.path.exists(npz_file):
+                    found = False
+                    break
+
+                latents, _, _ = train_util.load_latents_from_disk(npz_file)
+                if latents is None:  # old version
+                    found = False
+                    break
+
+                if latents.shape[1] != reso[1] // 8 or latents.shape[2] != reso[0] // 8:  # latentsのshapeを確認
+                    found = False
+                    break
+            if found:
+                continue
+
+        # 画像をリサイズしてトリミングする
+        # PILにinter_areaがないのでcv2で……
+        image = np.array(image)
+        if resized_size[0] != image.shape[1] or resized_size[1] != image.shape[0]:  # リサイズ処理が必要？
+            image = cv2.resize(image, resized_size, interpolation=cv2.INTER_AREA)
+
+        trim_left = 0
+        if resized_size[0] > reso[0]:
+            trim_size = resized_size[0] - reso[0]
+            image = image[:, trim_size // 2 : trim_size // 2 + reso[0]]
+            trim_left = trim_size // 2
+
+        trim_top = 0
+        if resized_size[1] > reso[1]:
+            trim_size = resized_size[1] - reso[1]
+            image = image[trim_size // 2 : trim_size // 2 + reso[1]]
+            trim_top = trim_size // 2
+
+        original_size_wh = (resized_size[0], resized_size[1])
+        # target_size_wh = (reso[0], reso[1])
+        crop_left_top = (trim_left, trim_top)
+
+        assert (
+            image.shape[0] == reso[1] and image.shape[1] == reso[0]
+        ), f"internal error, illegal trimmed size: {image.shape}, {reso}"
+
+        # # debug
+        # cv2.imwrite(f"r:\\test\\img_{len(img_ar_errors)}.jpg", image[:, :, ::-1])
+
+        # バッチへ追加
+        bucket_manager.add_image(reso, (image_key, image, original_size_wh, crop_left_top))
+
+        # バッチを推論するか判定して推論する
+        process_batch(False)
+
+    # 残りを処理する
+    process_batch(True)
+
+    bucket_manager.sort()
+    for i, reso in enumerate(bucket_manager.resos):
+        count = bucket_counts.get(reso, 0)
+        if count > 0:
+            print(f"bucket {i} {reso}: {count}")
+    img_ar_errors = np.array(img_ar_errors)
+    print(f"mean ar error: {np.mean(img_ar_errors)}")
+
+    # metadataを書き出して終わり
+    print(f"writing metadata: {args.out_json}")
+    with open(args.out_json, "wt", encoding="utf-8") as f:
+        json.dump(metadata, f, indent=2)
+    print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("train_data_dir", type=str, help="directory for train images / 学習画像データのディレクトリ")
+    parser.add_argument("in_json", type=str, help="metadata file to input / 読み込むメタデータファイル")
+    parser.add_argument("out_json", type=str, help="metadata file to output / メタデータファイル書き出し先")
+    parser.add_argument("model_name_or_path", type=str, help="model name or path to encode latents / latentを取得するためのモデル")
+    parser.add_argument("--v2", action="store_true", help="not used (for backward compatibility) / 使用されません（互換性のため残してあります）")
+    parser.add_argument("--batch_size", type=int, default=1, help="batch size in inference / 推論時のバッチサイズ")
+    parser.add_argument(
+        "--max_data_loader_n_workers",
+        type=int,
+        default=None,
+        help="enable image reading by DataLoader with this number of workers (faster) / DataLoaderによる画像読み込みを有効にしてこのワーカー数を適用する（読み込みを高速化）",
+    )
+    parser.add_argument(
+        "--max_resolution",
+        type=str,
+        default="512,512",
+        help="max resolution in fine tuning (width,height) / fine tuning時の最大画像サイズ 「幅,高さ」（使用メモリ量に関係します）",
+    )
+    parser.add_argument("--min_bucket_reso", type=int, default=256, help="minimum resolution for buckets / bucketの最小解像度")
+    parser.add_argument("--max_bucket_reso", type=int, default=1024, help="maximum resolution for buckets / bucketの最小解像度")
+    parser.add_argument(
+        "--bucket_reso_steps",
+        type=int,
+        default=64,
+        help="steps of resolution for buckets, divisible by 8 is recommended / bucketの解像度の単位、8で割り切れる値を推奨します",
+    )
+    parser.add_argument(
+        "--bucket_no_upscale", action="store_true", help="make bucket for each image without upscaling / 画像を拡大せずbucketを作成します"
+    )
+    parser.add_argument(
+        "--mixed_precision", type=str, default="no", choices=["no", "fp16", "bf16"], help="use mixed precision / 混合精度を使う場合、その精度"
+    )
+    parser.add_argument(
+        "--full_path",
+        action="store_true",
+        help="use full path as image-key in metadata (supports multiple directories) / メタデータで画像キーをフルパスにする（複数の学習画像ディレクトリに対応）",
+    )
+    parser.add_argument(
+        "--flip_aug", action="store_true", help="flip augmentation, save latents for flipped images / 左右反転した画像もlatentを取得、保存する"
+    )
+    parser.add_argument(
+        "--skip_existing",
+        action="store_true",
+        help="skip images if npz already exists (both normal and flipped exists if flip_aug is enabled) / npzが既に存在する画像をスキップする（flip_aug有効時は通常、反転の両方が存在する画像をスキップ）",
+    )
+    parser.add_argument(
+        "--recursive",
+        action="store_true",
+        help="recursively look for training tags in all child folders of train_data_dir / train_data_dirのすべての子フォルダにある学習タグを再帰的に探す",
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    main(args)
diff --git a/finetune/tag_images_by_wd14_tagger.py b/finetune/tag_images_by_wd14_tagger.py
new file mode 100644
index 0000000000000000000000000000000000000000..91e4f573e1a431766da81e80fa15daeb233df0c2
--- /dev/null
+++ b/finetune/tag_images_by_wd14_tagger.py
@@ -0,0 +1,303 @@
+import argparse
+import csv
+import glob
+import os
+
+from PIL import Image
+import cv2
+from tqdm import tqdm
+import numpy as np
+from tensorflow.keras.models import load_model
+from huggingface_hub import hf_hub_download
+import torch
+from pathlib import Path
+
+import library.train_util as train_util
+
+# from wd14 tagger
+IMAGE_SIZE = 448
+
+# wd-v1-4-swinv2-tagger-v2 / wd-v1-4-vit-tagger / wd-v1-4-vit-tagger-v2/ wd-v1-4-convnext-tagger / wd-v1-4-convnext-tagger-v2
+DEFAULT_WD14_TAGGER_REPO = "SmilingWolf/wd-v1-4-convnext-tagger-v2"
+FILES = ["keras_metadata.pb", "saved_model.pb", "selected_tags.csv"]
+SUB_DIR = "variables"
+SUB_DIR_FILES = ["variables.data-00000-of-00001", "variables.index"]
+CSV_FILE = FILES[-1]
+
+
+def preprocess_image(image):
+    image = np.array(image)
+    image = image[:, :, ::-1]  # RGB->BGR
+
+    # pad to square
+    size = max(image.shape[0:2])
+    pad_x = size - image.shape[1]
+    pad_y = size - image.shape[0]
+    pad_l = pad_x // 2
+    pad_t = pad_y // 2
+    image = np.pad(image, ((pad_t, pad_y - pad_t), (pad_l, pad_x - pad_l), (0, 0)), mode="constant", constant_values=255)
+
+    interp = cv2.INTER_AREA if size > IMAGE_SIZE else cv2.INTER_LANCZOS4
+    image = cv2.resize(image, (IMAGE_SIZE, IMAGE_SIZE), interpolation=interp)
+
+    image = image.astype(np.float32)
+    return image
+
+
+class ImageLoadingPrepDataset(torch.utils.data.Dataset):
+    def __init__(self, image_paths):
+        self.images = image_paths
+
+    def __len__(self):
+        return len(self.images)
+
+    def __getitem__(self, idx):
+        img_path = str(self.images[idx])
+
+        try:
+            image = Image.open(img_path).convert("RGB")
+            image = preprocess_image(image)
+            tensor = torch.tensor(image)
+        except Exception as e:
+            print(f"Could not load image path / 画像を読み込めません: {img_path}, error: {e}")
+            return None
+
+        return (tensor, img_path)
+
+
+def collate_fn_remove_corrupted(batch):
+    """Collate function that allows to remove corrupted examples in the
+    dataloader. It expects that the dataloader returns 'None' when that occurs.
+    The 'None's in the batch are removed.
+    """
+    # Filter out all the Nones (corrupted examples)
+    batch = list(filter(lambda x: x is not None, batch))
+    return batch
+
+
+def main(args):
+    # hf_hub_downloadをそのまま使うとsymlink関係で問題があるらしいので、キャッシュディレクトリとforce_filenameを指定してなんとかする
+    # depreacatedの警告が出るけどなくなったらその時
+    # https://github.com/toriato/stable-diffusion-webui-wd14-tagger/issues/22
+    if not os.path.exists(args.model_dir) or args.force_download:
+        print(f"downloading wd14 tagger model from hf_hub. id: {args.repo_id}")
+        for file in FILES:
+            hf_hub_download(args.repo_id, file, cache_dir=args.model_dir, force_download=True, force_filename=file)
+        for file in SUB_DIR_FILES:
+            hf_hub_download(
+                args.repo_id,
+                file,
+                subfolder=SUB_DIR,
+                cache_dir=os.path.join(args.model_dir, SUB_DIR),
+                force_download=True,
+                force_filename=file,
+            )
+    else:
+        print("using existing wd14 tagger model")
+
+    # 画像を読み込む
+    model = load_model(args.model_dir)
+
+    # label_names = pd.read_csv("2022_0000_0899_6549/selected_tags.csv")
+    # 依存ライブラリを増やしたくないので自力で読むよ
+
+    with open(os.path.join(args.model_dir, CSV_FILE), "r", encoding="utf-8") as f:
+        reader = csv.reader(f)
+        l = [row for row in reader]
+        header = l[0]  # tag_id,name,category,count
+        rows = l[1:]
+    assert header[0] == "tag_id" and header[1] == "name" and header[2] == "category", f"unexpected csv format: {header}"
+
+    general_tags = [row[1] for row in rows[1:] if row[2] == "0"]
+    character_tags = [row[1] for row in rows[1:] if row[2] == "4"]
+
+    # 画像を読み込む
+
+    train_data_dir_path = Path(args.train_data_dir)
+    image_paths = train_util.glob_images_pathlib(train_data_dir_path, args.recursive)
+    print(f"found {len(image_paths)} images.")
+
+    tag_freq = {}
+
+    undesired_tags = set(args.undesired_tags.split(","))
+
+    def run_batch(path_imgs):
+        imgs = np.array([im for _, im in path_imgs])
+
+        probs = model(imgs, training=False)
+        probs = probs.numpy()
+
+        for (image_path, _), prob in zip(path_imgs, probs):
+            # 最初の4つはratingなので無視する
+            # # First 4 labels are actually ratings: pick one with argmax
+            # ratings_names = label_names[:4]
+            # rating_index = ratings_names["probs"].argmax()
+            # found_rating = ratings_names[rating_index: rating_index + 1][["name", "probs"]]
+
+            # それ以降はタグなのでconfidenceがthresholdより高いものを追加する
+            # Everything else is tags: pick any where prediction confidence > threshold
+            combined_tags = []
+            general_tag_text = ""
+            character_tag_text = ""
+            for i, p in enumerate(prob[4:]):
+                if i < len(general_tags) and p >= args.general_threshold:
+                    tag_name = general_tags[i]
+                    if args.remove_underscore and len(tag_name) > 3:  # ignore emoji tags like >_< and ^_^
+                        tag_name = tag_name.replace("_", " ")
+
+                    if tag_name not in undesired_tags:
+                        tag_freq[tag_name] = tag_freq.get(tag_name, 0) + 1
+                        general_tag_text += ", " + tag_name
+                        combined_tags.append(tag_name)
+                elif i >= len(general_tags) and p >= args.character_threshold:
+                    tag_name = character_tags[i - len(general_tags)]
+                    if args.remove_underscore and len(tag_name) > 3:
+                        tag_name = tag_name.replace("_", " ")
+
+                    if tag_name not in undesired_tags:
+                        tag_freq[tag_name] = tag_freq.get(tag_name, 0) + 1
+                        character_tag_text += ", " + tag_name
+                        combined_tags.append(tag_name)
+
+            # 先頭のカンマを取る
+            if len(general_tag_text) > 0:
+                general_tag_text = general_tag_text[2:]
+            if len(character_tag_text) > 0:
+                character_tag_text = character_tag_text[2:]
+
+            tag_text = ", ".join(combined_tags)
+
+            with open(os.path.splitext(image_path)[0] + args.caption_extension, "wt", encoding="utf-8") as f:
+                f.write(tag_text + "\n")
+                if args.debug:
+                    print(f"\n{image_path}:\n  Character tags: {character_tag_text}\n  General tags: {general_tag_text}")
+
+    # 読み込みの高速化のためにDataLoaderを使うオプション
+    if args.max_data_loader_n_workers is not None:
+        dataset = ImageLoadingPrepDataset(image_paths)
+        data = torch.utils.data.DataLoader(
+            dataset,
+            batch_size=args.batch_size,
+            shuffle=False,
+            num_workers=args.max_data_loader_n_workers,
+            collate_fn=collate_fn_remove_corrupted,
+            drop_last=False,
+        )
+    else:
+        data = [[(None, ip)] for ip in image_paths]
+
+    b_imgs = []
+    for data_entry in tqdm(data, smoothing=0.0):
+        for data in data_entry:
+            if data is None:
+                continue
+
+            image, image_path = data
+            if image is not None:
+                image = image.detach().numpy()
+            else:
+                try:
+                    image = Image.open(image_path)
+                    if image.mode != "RGB":
+                        image = image.convert("RGB")
+                    image = preprocess_image(image)
+                except Exception as e:
+                    print(f"Could not load image path / 画像を読み込めません: {image_path}, error: {e}")
+                    continue
+            b_imgs.append((image_path, image))
+
+            if len(b_imgs) >= args.batch_size:
+                b_imgs = [(str(image_path), image) for image_path, image in b_imgs]  # Convert image_path to string
+                run_batch(b_imgs)
+                b_imgs.clear()
+
+    if len(b_imgs) > 0:
+        b_imgs = [(str(image_path), image) for image_path, image in b_imgs]  # Convert image_path to string
+        run_batch(b_imgs)
+
+    if args.frequency_tags:
+        sorted_tags = sorted(tag_freq.items(), key=lambda x: x[1], reverse=True)
+        print("\nTag frequencies:")
+        for tag, freq in sorted_tags:
+            print(f"{tag}: {freq}")
+
+    print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("train_data_dir", type=str, help="directory for train images / 学習画像データのディレクトリ")
+    parser.add_argument(
+        "--repo_id",
+        type=str,
+        default=DEFAULT_WD14_TAGGER_REPO,
+        help="repo id for wd14 tagger on Hugging Face / Hugging Faceのwd14 taggerのリポジトリID",
+    )
+    parser.add_argument(
+        "--model_dir",
+        type=str,
+        default="wd14_tagger_model",
+        help="directory to store wd14 tagger model / wd14 taggerのモデルを格納するディレクトリ",
+    )
+    parser.add_argument(
+        "--force_download", action="store_true", help="force downloading wd14 tagger models / wd14 taggerのモデルを再ダウンロードします"
+    )
+    parser.add_argument("--batch_size", type=int, default=1, help="batch size in inference / 推論時のバッチサイズ")
+    parser.add_argument(
+        "--max_data_loader_n_workers",
+        type=int,
+        default=None,
+        help="enable image reading by DataLoader with this number of workers (faster) / DataLoaderによる画像読み込みを有効にしてこのワーカー数を適用する（読み込みを高速化）",
+    )
+    parser.add_argument(
+        "--caption_extention",
+        type=str,
+        default=None,
+        help="extension of caption file (for backward compatibility) / 出力されるキャプションファイルの拡張子（スペルミスしていたのを残してあります）",
+    )
+    parser.add_argument("--caption_extension", type=str, default=".txt", help="extension of caption file / 出力されるキャプションファイルの拡張子")
+    parser.add_argument("--thresh", type=float, default=0.35, help="threshold of confidence to add a tag / タグを追加するか判定する閾値")
+    parser.add_argument(
+        "--general_threshold",
+        type=float,
+        default=None,
+        help="threshold of confidence to add a tag for general category, same as --thresh if omitted / generalカテゴリのタグを追加するための確信度の閾値、省略時は --thresh と同じ",
+    )
+    parser.add_argument(
+        "--character_threshold",
+        type=float,
+        default=None,
+        help="threshold of confidence to add a tag for character category, same as --thres if omitted / characterカテゴリのタグを追加するための確信度の閾値、省略時は --thresh と同じ",
+    )
+    parser.add_argument("--recursive", action="store_true", help="search for images in subfolders recursively / サブフォルダを再帰的に検索する")
+    parser.add_argument(
+        "--remove_underscore",
+        action="store_true",
+        help="replace underscores with spaces in the output tags / 出力されるタグのアンダースコアをスペースに置き換える",
+    )
+    parser.add_argument("--debug", action="store_true", help="debug mode")
+    parser.add_argument(
+        "--undesired_tags",
+        type=str,
+        default="",
+        help="comma-separated list of undesired tags to remove from the output / 出力から除外したいタグのカンマ区切りのリスト",
+    )
+    parser.add_argument("--frequency_tags", action="store_true", help="Show frequency of tags for images / 画像ごとのタグの出現頻度を表示する")
+
+    return parser
+
+if __name__ == "__main__":
+    parser = setup_parser()
+    
+    args = parser.parse_args()
+
+    # スペルミスしていたオプションを復元する
+    if args.caption_extention is not None:
+        args.caption_extension = args.caption_extention
+
+    if args.general_threshold is None:
+        args.general_threshold = args.thresh
+    if args.character_threshold is None:
+        args.character_threshold = args.thresh
+
+    main(args)
diff --git a/finetune_gui.py b/finetune_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..5253ac07c92fa9b92c804fe5ef799246748e9f44
--- /dev/null
+++ b/finetune_gui.py
@@ -0,0 +1,1050 @@
+import gradio as gr
+import json
+import math
+import os
+import subprocess
+import pathlib
+import argparse
+from datetime import datetime
+from library.common_gui import (
+    get_folder_path,
+    get_file_path,
+    get_saveasfile_path,
+    save_inference_file,
+    run_cmd_advanced_training,
+    color_aug_changed,
+    run_cmd_training,
+    update_my_data,
+    check_if_model_exist,
+    SaveConfigFile,
+    save_to_file
+)
+from library.class_configuration_file import ConfigurationFile
+from library.class_source_model import SourceModel
+from library.class_basic_training import BasicTraining
+from library.class_advanced_training import AdvancedTraining
+from library.class_sdxl_parameters import SDXLParameters
+from library.tensorboard_gui import (
+    gradio_tensorboard,
+    start_tensorboard,
+    stop_tensorboard,
+)
+from library.utilities import utilities_tab
+from library.class_sample_images import SampleImages, run_cmd_sample
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+# from easygui import msgbox
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def save_configuration(
+    save_as,
+    file_path,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl_checkbox,
+    train_dir,
+    image_folder,
+    output_dir,
+    logging_dir,
+    max_resolution,
+    min_bucket_reso,
+    max_bucket_reso,
+    batch_size,
+    flip_aug,
+    caption_metadata_filename,
+    latent_metadata_filename,
+    full_path,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    dataset_repeats,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    train_text_encoder,
+    full_bf16,
+    create_caption,
+    create_buckets,
+    save_model_as,
+    caption_extension,
+    # use_8bit_adam,
+    xformers,
+    clip_skip,
+    save_state,
+    resume,
+    gradient_checkpointing,
+    gradient_accumulation_steps,
+    mem_eff_attn,
+    shuffle_caption,
+    output_name,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    full_fp16,
+    color_aug,
+    model_list,
+    cache_latents,
+    cache_latents_to_disk,
+    use_latent_files,
+    keep_tokens,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    weighted_captions,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    sdxl_cache_text_encoder_outputs,
+    sdxl_no_half_vae,
+    min_timestep,
+    max_timestep,
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+
+    original_file_path = file_path
+
+    save_as_bool = True if save_as.get('label') == 'True' else False
+
+    if save_as_bool:
+        log.info('Save as...')
+        file_path = get_saveasfile_path(file_path)
+    else:
+        log.info('Save...')
+        if file_path == None or file_path == '':
+            file_path = get_saveasfile_path(file_path)
+
+    # log.info(file_path)
+
+    if file_path == None or file_path == '':
+        return original_file_path  # In case a file_path was provided and the user decide to cancel the open action
+
+    # Extract the destination directory from the file path
+    destination_directory = os.path.dirname(file_path)
+
+    # Create the destination directory if it doesn't exist
+    if not os.path.exists(destination_directory):
+        os.makedirs(destination_directory)
+
+    SaveConfigFile(parameters=parameters, file_path=file_path, exclusion=['file_path', 'save_as'])
+
+    return file_path
+
+
+def open_configuration(
+    ask_for_file,
+    file_path,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl_checkbox,
+    train_dir,
+    image_folder,
+    output_dir,
+    logging_dir,
+    max_resolution,
+    min_bucket_reso,
+    max_bucket_reso,
+    batch_size,
+    flip_aug,
+    caption_metadata_filename,
+    latent_metadata_filename,
+    full_path,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    dataset_repeats,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    train_text_encoder,
+    full_bf16,
+    create_caption,
+    create_buckets,
+    save_model_as,
+    caption_extension,
+    # use_8bit_adam,
+    xformers,
+    clip_skip,
+    save_state,
+    resume,
+    gradient_checkpointing,
+    gradient_accumulation_steps,
+    mem_eff_attn,
+    shuffle_caption,
+    output_name,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    full_fp16,
+    color_aug,
+    model_list,
+    cache_latents,
+    cache_latents_to_disk,
+    use_latent_files,
+    keep_tokens,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    weighted_captions,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    sdxl_cache_text_encoder_outputs,
+    sdxl_no_half_vae,
+    min_timestep,
+    max_timestep,
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+
+    ask_for_file = True if ask_for_file.get('label') == 'True' else False
+
+    original_file_path = file_path
+
+    if ask_for_file:
+        file_path = get_file_path(file_path)
+
+    if not file_path == '' and not file_path == None:
+        # load variables from JSON file
+        with open(file_path, 'r') as f:
+            my_data = json.load(f)
+            log.info('Loading config...')
+            # Update values to fix deprecated use_8bit_adam checkbox and set appropriate optimizer if it is set to True
+            my_data = update_my_data(my_data)
+    else:
+        file_path = original_file_path  # In case a file_path was provided and the user decide to cancel the open action
+        my_data = {}
+
+    values = [file_path]
+    for key, value in parameters:
+        # Set the value in the dictionary to the corresponding value in `my_data`, or the default value if not found
+        if not key in ['ask_for_file', 'file_path']:
+            values.append(my_data.get(key, value))
+    return tuple(values)
+
+
+def train_model(
+    headless,
+    print_only,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl_checkbox,
+    train_dir,
+    image_folder,
+    output_dir,
+    logging_dir,
+    max_resolution,
+    min_bucket_reso,
+    max_bucket_reso,
+    batch_size,
+    flip_aug,
+    caption_metadata_filename,
+    latent_metadata_filename,
+    full_path,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    dataset_repeats,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    train_text_encoder,
+    full_bf16,
+    generate_caption_database,
+    generate_image_buckets,
+    save_model_as,
+    caption_extension,
+    # use_8bit_adam,
+    xformers,
+    clip_skip,
+    save_state,
+    resume,
+    gradient_checkpointing,
+    gradient_accumulation_steps,
+    mem_eff_attn,
+    shuffle_caption,
+    output_name,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    full_fp16,
+    color_aug,
+    model_list,  # Keep this. Yes, it is unused here but required given the common list used
+    cache_latents,
+    cache_latents_to_disk,
+    use_latent_files,
+    keep_tokens,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    weighted_captions,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    sdxl_cache_text_encoder_outputs,
+    sdxl_no_half_vae,
+    min_timestep,
+    max_timestep,
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+    
+    print_only_bool = True if print_only.get('label') == 'True' else False
+    log.info(f'Start Finetuning...')
+
+    headless_bool = True if headless.get('label') == 'True' else False
+
+    if check_if_model_exist(
+        output_name, output_dir, save_model_as, headless_bool
+    ):
+        return
+
+    # if float(noise_offset) > 0 and (
+    #     multires_noise_iterations > 0 or multires_noise_discount > 0
+    # ):
+    #     output_message(
+    #         msg="noise offset and multires_noise can't be set at the same time. Only use one or the other.",
+    #         title='Error',
+    #         headless=headless_bool,
+    #     )
+    #     return
+
+    # if optimizer == 'Adafactor' and lr_warmup != '0':
+    #     output_message(
+    #         msg="Warning: lr_scheduler is set to 'Adafactor', so 'LR warmup (% of steps)' will be considered 0.",
+    #         title='Warning',
+    #         headless=headless_bool,
+    #     )
+    #     lr_warmup = '0'
+
+    # create caption json file
+    if generate_caption_database:
+        if not os.path.exists(train_dir):
+            os.mkdir(train_dir)
+
+        run_cmd = f'{PYTHON} finetune/merge_captions_to_metadata.py'
+        if caption_extension == '':
+            run_cmd += f' --caption_extension=".caption"'
+        else:
+            run_cmd += f' --caption_extension={caption_extension}'
+        run_cmd += f' "{image_folder}"'
+        run_cmd += f' "{train_dir}/{caption_metadata_filename}"'
+        if full_path:
+            run_cmd += f' --full_path'
+
+        log.info(run_cmd)
+
+        if not print_only_bool:
+            # Run the command
+            if os.name == 'posix':
+                os.system(run_cmd)
+            else:
+                subprocess.run(run_cmd)
+
+    # create images buckets
+    if generate_image_buckets:
+        run_cmd = f'{PYTHON} finetune/prepare_buckets_latents.py'
+        run_cmd += f' "{image_folder}"'
+        run_cmd += f' "{train_dir}/{caption_metadata_filename}"'
+        run_cmd += f' "{train_dir}/{latent_metadata_filename}"'
+        run_cmd += f' "{pretrained_model_name_or_path}"'
+        run_cmd += f' --batch_size={batch_size}'
+        run_cmd += f' --max_resolution={max_resolution}'
+        run_cmd += f' --min_bucket_reso={min_bucket_reso}'
+        run_cmd += f' --max_bucket_reso={max_bucket_reso}'
+        run_cmd += f' --mixed_precision={mixed_precision}'
+        # if flip_aug:
+        #     run_cmd += f' --flip_aug'
+        if full_path:
+            run_cmd += f' --full_path'
+
+        log.info(run_cmd)
+
+        if not print_only_bool:
+            # Run the command
+            if os.name == 'posix':
+                os.system(run_cmd)
+            else:
+                subprocess.run(run_cmd)
+
+    image_num = len(
+        [
+            f
+            for f, lower_f in (
+                (file, file.lower()) for file in os.listdir(image_folder)
+            )
+            if lower_f.endswith(('.jpg', '.jpeg', '.png', '.webp'))
+        ]
+    )
+    log.info(f'image_num = {image_num}')
+
+    repeats = int(image_num) * int(dataset_repeats)
+    log.info(f'repeats = {str(repeats)}')
+
+    # calculate max_train_steps
+    max_train_steps = int(
+        math.ceil(
+            float(repeats)
+            / int(train_batch_size)
+            / int(gradient_accumulation_steps)
+            * int(epoch)
+        )
+    )
+
+    # Divide by two because flip augmentation create two copied of the source images
+    if flip_aug:
+        max_train_steps = int(math.ceil(float(max_train_steps) / 2))
+
+    log.info(f'max_train_steps = {max_train_steps}')
+
+    lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+    log.info(f'lr_warmup_steps = {lr_warmup_steps}')
+
+    run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process}'
+    if sdxl_checkbox:
+        run_cmd += f' "./sdxl_train.py"'
+    else:
+        run_cmd += f' "./fine_tune.py"'
+
+    if v2:
+        run_cmd += ' --v2'
+    if v_parameterization:
+        run_cmd += ' --v_parameterization'
+    if train_text_encoder:
+        run_cmd += ' --train_text_encoder'
+    if full_bf16:
+        run_cmd += ' --full_bf16'
+    if weighted_captions:
+        run_cmd += ' --weighted_captions'
+    run_cmd += (
+        f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"'
+    )
+    if use_latent_files == 'Yes':
+        run_cmd += f' --in_json="{train_dir}/{latent_metadata_filename}"'
+    else:
+        run_cmd += f' --in_json="{train_dir}/{caption_metadata_filename}"'
+    run_cmd += f' --train_data_dir="{image_folder}"'
+    run_cmd += f' --output_dir="{output_dir}"'
+    if not logging_dir == '':
+        run_cmd += f' --logging_dir="{logging_dir}"'
+    run_cmd += f' --dataset_repeats={dataset_repeats}'
+    run_cmd += f' --learning_rate={learning_rate}'
+
+    run_cmd += ' --enable_bucket'
+    run_cmd += f' --resolution="{max_resolution}"'
+    run_cmd += f' --min_bucket_reso={min_bucket_reso}'
+    run_cmd += f' --max_bucket_reso={max_bucket_reso}'
+
+    if not save_model_as == 'same as source model':
+        run_cmd += f' --save_model_as={save_model_as}'
+    if int(gradient_accumulation_steps) > 1:
+        run_cmd += f' --gradient_accumulation_steps={int(gradient_accumulation_steps)}'
+    # if save_state:
+    #     run_cmd += ' --save_state'
+    # if not resume == '':
+    #     run_cmd += f' --resume={resume}'
+    if not output_name == '':
+        run_cmd += f' --output_name="{output_name}"'
+    if int(max_token_length) > 75:
+        run_cmd += f' --max_token_length={max_token_length}'
+        
+    if sdxl_cache_text_encoder_outputs:
+        run_cmd += f' --cache_text_encoder_outputs'
+        
+    if sdxl_no_half_vae:
+        run_cmd += f' --no_half_vae'
+
+    run_cmd += run_cmd_training(
+        learning_rate=learning_rate,
+        lr_scheduler=lr_scheduler,
+        lr_warmup_steps=lr_warmup_steps,
+        train_batch_size=train_batch_size,
+        max_train_steps=max_train_steps,
+        save_every_n_epochs=save_every_n_epochs,
+        mixed_precision=mixed_precision,
+        save_precision=save_precision,
+        seed=seed,
+        caption_extension=caption_extension,
+        cache_latents=cache_latents,
+        cache_latents_to_disk=cache_latents_to_disk,
+        optimizer=optimizer,
+        optimizer_args=optimizer_args,
+    )
+
+    run_cmd += run_cmd_advanced_training(
+        max_train_epochs=max_train_epochs,
+        max_data_loader_n_workers=max_data_loader_n_workers,
+        max_token_length=max_token_length,
+        resume=resume,
+        save_state=save_state,
+        mem_eff_attn=mem_eff_attn,
+        clip_skip=clip_skip,
+        flip_aug=flip_aug,
+        color_aug=color_aug,
+        shuffle_caption=shuffle_caption,
+        gradient_checkpointing=gradient_checkpointing,
+        full_fp16=full_fp16,
+        xformers=xformers,
+        # use_8bit_adam=use_8bit_adam,
+        keep_tokens=keep_tokens,
+        persistent_data_loader_workers=persistent_data_loader_workers,
+        bucket_no_upscale=bucket_no_upscale,
+        random_crop=random_crop,
+        bucket_reso_steps=bucket_reso_steps,
+        caption_dropout_every_n_epochs=caption_dropout_every_n_epochs,
+        caption_dropout_rate=caption_dropout_rate,
+        noise_offset_type=noise_offset_type,
+        noise_offset=noise_offset,
+        adaptive_noise_scale=adaptive_noise_scale,
+        multires_noise_iterations=multires_noise_iterations,
+        multires_noise_discount=multires_noise_discount,
+        additional_parameters=additional_parameters,
+        vae_batch_size=vae_batch_size,
+        min_snr_gamma=min_snr_gamma,
+        save_every_n_steps=save_every_n_steps,
+        save_last_n_steps=save_last_n_steps,
+        save_last_n_steps_state=save_last_n_steps_state,
+        use_wandb=use_wandb,
+        wandb_api_key=wandb_api_key,
+        scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred,
+        min_timestep=min_timestep,
+        max_timestep=max_timestep,
+    )
+
+    run_cmd += run_cmd_sample(
+        sample_every_n_steps,
+        sample_every_n_epochs,
+        sample_sampler,
+        sample_prompts,
+        output_dir,
+    )
+
+    if print_only_bool:
+        log.warning(
+            'Here is the trainer command as a reference. It will not be executed:\n'
+        )
+        print(run_cmd)
+        
+        save_to_file(run_cmd)
+    else:
+        # Saving config file for model
+        current_datetime = datetime.now()
+        formatted_datetime = current_datetime.strftime("%Y%m%d-%H%M%S")
+        file_path = os.path.join(output_dir, f'{output_name}_{formatted_datetime}.json')
+        
+        log.info(f'Saving training config to {file_path}...')
+
+        SaveConfigFile(parameters=parameters, file_path=file_path, exclusion=['file_path', 'save_as', 'headless', 'print_only'])
+        
+        log.info(run_cmd)
+
+        # Run the command
+        if os.name == 'posix':
+            os.system(run_cmd)
+        else:
+            subprocess.run(run_cmd)
+
+        # check if output_dir/last is a folder... therefore it is a diffuser model
+        last_dir = pathlib.Path(f'{output_dir}/{output_name}')
+
+        if not last_dir.is_dir():
+            # Copy inference model for v2 if required
+            save_inference_file(
+                output_dir, v2, v_parameterization, output_name
+            )
+
+
+def remove_doublequote(file_path):
+    if file_path != None:
+        file_path = file_path.replace('"', '')
+
+    return file_path
+
+
+def finetune_tab(headless=False):
+    dummy_db_true = gr.Label(value=True, visible=False)
+    dummy_db_false = gr.Label(value=False, visible=False)
+    dummy_headless = gr.Label(value=headless, visible=False)
+    with gr.Tab('Training'):
+        gr.Markdown('Train a custom model using kohya finetune python code...')
+
+        # Setup Configuration Files Gradio
+        config = ConfigurationFile(headless)
+
+        source_model = SourceModel(headless=headless)
+
+        with gr.Tab('Folders'):
+            with gr.Row():
+                train_dir = gr.Textbox(
+                    label='Training config folder',
+                    placeholder='folder where the training configuration files will be saved',
+                )
+                train_dir_folder = gr.Button(
+                    folder_symbol,
+                    elem_id='open_folder_small',
+                    visible=(not headless),
+                )
+                train_dir_folder.click(
+                    get_folder_path,
+                    outputs=train_dir,
+                    show_progress=False,
+                )
+
+                image_folder = gr.Textbox(
+                    label='Training Image folder',
+                    placeholder='folder where the training images are located',
+                )
+                image_folder_input_folder = gr.Button(
+                    folder_symbol,
+                    elem_id='open_folder_small',
+                    visible=(not headless),
+                )
+                image_folder_input_folder.click(
+                    get_folder_path,
+                    outputs=image_folder,
+                    show_progress=False,
+                )
+            with gr.Row():
+                output_dir = gr.Textbox(
+                    label='Model output folder',
+                    placeholder='folder where the model will be saved',
+                )
+                output_dir_input_folder = gr.Button(
+                    folder_symbol,
+                    elem_id='open_folder_small',
+                    visible=(not headless),
+                )
+                output_dir_input_folder.click(
+                    get_folder_path,
+                    outputs=output_dir,
+                    show_progress=False,
+                )
+
+                logging_dir = gr.Textbox(
+                    label='Logging folder',
+                    placeholder='Optional: enable logging and output TensorBoard log to this folder',
+                )
+                logging_dir_input_folder = gr.Button(
+                    folder_symbol,
+                    elem_id='open_folder_small',
+                    visible=(not headless),
+                )
+                logging_dir_input_folder.click(
+                    get_folder_path,
+                    outputs=logging_dir,
+                    show_progress=False,
+                )
+            with gr.Row():
+                output_name = gr.Textbox(
+                    label='Model output name',
+                    placeholder='Name of the model to output',
+                    value='last',
+                    interactive=True,
+                )
+            train_dir.change(
+                remove_doublequote,
+                inputs=[train_dir],
+                outputs=[train_dir],
+            )
+            image_folder.change(
+                remove_doublequote,
+                inputs=[image_folder],
+                outputs=[image_folder],
+            )
+            output_dir.change(
+                remove_doublequote,
+                inputs=[output_dir],
+                outputs=[output_dir],
+            )
+        with gr.Tab('Dataset preparation'):
+            with gr.Row():
+                max_resolution = gr.Textbox(
+                    label='Resolution (width,height)', value='512,512'
+                )
+                min_bucket_reso = gr.Textbox(
+                    label='Min bucket resolution', value='256'
+                )
+                max_bucket_reso = gr.Textbox(
+                    label='Max bucket resolution', value='1024'
+                )
+                batch_size = gr.Textbox(label='Batch size', value='1')
+            with gr.Row():
+                create_caption = gr.Checkbox(
+                    label='Generate caption metadata', value=True
+                )
+                create_buckets = gr.Checkbox(
+                    label='Generate image buckets metadata', value=True
+                )
+                use_latent_files = gr.Dropdown(
+                    label='Use latent files',
+                    choices=[
+                        'No',
+                        'Yes',
+                    ],
+                    value='Yes',
+                )
+            with gr.Accordion('Advanced parameters', open=False):
+                with gr.Row():
+                    caption_metadata_filename = gr.Textbox(
+                        label='Caption metadata filename', value='meta_cap.json'
+                    )
+                    latent_metadata_filename = gr.Textbox(
+                        label='Latent metadata filename', value='meta_lat.json'
+                    )
+                with gr.Row():
+                    full_path = gr.Checkbox(label='Use full path', value=True)
+                    weighted_captions = gr.Checkbox(
+                        label='Weighted captions', value=False
+                    )
+        with gr.Tab('Parameters'):
+            basic_training = BasicTraining(learning_rate_value='1e-5', finetuning=True)
+            
+            # Add SDXL Parameters
+            sdxl_params = SDXLParameters(source_model.sdxl_checkbox)
+            
+            with gr.Row():
+                dataset_repeats = gr.Textbox(label='Dataset repeats', value=40)
+                train_text_encoder = gr.Checkbox(
+                    label='Train text encoder', value=True
+                )
+                full_bf16 = gr.Checkbox(
+                    label='Full bf16', value = False
+                )
+            with gr.Accordion('Advanced parameters', open=False):
+                with gr.Row():
+                    gradient_accumulation_steps = gr.Number(
+                        label='Gradient accumulate steps', value='1'
+                    )
+                advanced_training = AdvancedTraining(headless=headless, finetuning=True)
+                advanced_training.color_aug.change(
+                    color_aug_changed,
+                    inputs=[advanced_training.color_aug],
+                    outputs=[basic_training.cache_latents],  # Not applicable to fine_tune.py
+                )
+
+            sample = SampleImages()
+
+        button_run = gr.Button('Train model', variant='primary')
+
+        button_print = gr.Button('Print training command')
+
+        # Setup gradio tensorboard buttons
+        button_start_tensorboard, button_stop_tensorboard = gradio_tensorboard()
+
+        button_start_tensorboard.click(
+            start_tensorboard,
+            inputs=logging_dir,
+        )
+
+        button_stop_tensorboard.click(
+            stop_tensorboard,
+            show_progress=False,
+        )
+
+        settings_list = [
+            source_model.pretrained_model_name_or_path,
+            source_model.v2,
+            source_model.v_parameterization,
+            source_model.sdxl_checkbox,
+            train_dir,
+            image_folder,
+            output_dir,
+            logging_dir,
+            max_resolution,
+            min_bucket_reso,
+            max_bucket_reso,
+            batch_size,
+            advanced_training.flip_aug,
+            caption_metadata_filename,
+            latent_metadata_filename,
+            full_path,
+            basic_training.learning_rate,
+            basic_training.lr_scheduler,
+            basic_training.lr_warmup,
+            dataset_repeats,
+            basic_training.train_batch_size,
+            basic_training.epoch,
+            basic_training.save_every_n_epochs,
+            basic_training.mixed_precision,
+            basic_training.save_precision,
+            basic_training.seed,
+            basic_training.num_cpu_threads_per_process,
+            train_text_encoder,
+            full_bf16,
+            create_caption,
+            create_buckets,
+            source_model.save_model_as,
+            basic_training.caption_extension,
+            advanced_training.xformers,
+            advanced_training.clip_skip,
+            advanced_training.save_state,
+            advanced_training.resume,
+            advanced_training.gradient_checkpointing,
+            gradient_accumulation_steps,
+            advanced_training.mem_eff_attn,
+            advanced_training.shuffle_caption,
+            output_name,
+            advanced_training.max_token_length,
+            advanced_training.max_train_epochs,
+            advanced_training.max_data_loader_n_workers,
+            advanced_training.full_fp16,
+            advanced_training.color_aug,
+            source_model.model_list,
+            basic_training.cache_latents,
+            basic_training.cache_latents_to_disk,
+            use_latent_files,
+            advanced_training.keep_tokens,
+            advanced_training.persistent_data_loader_workers,
+            advanced_training.bucket_no_upscale,
+            advanced_training.random_crop,
+            advanced_training.bucket_reso_steps,
+            advanced_training.caption_dropout_every_n_epochs,
+            advanced_training.caption_dropout_rate,
+            basic_training.optimizer,
+            basic_training.optimizer_args,
+            advanced_training.noise_offset_type,
+            advanced_training.noise_offset,
+            advanced_training.adaptive_noise_scale,
+            advanced_training.multires_noise_iterations,
+            advanced_training.multires_noise_discount,
+            sample.sample_every_n_steps,
+            sample.sample_every_n_epochs,
+            sample.sample_sampler,
+            sample.sample_prompts,
+            advanced_training.additional_parameters,
+            advanced_training.vae_batch_size,
+            advanced_training.min_snr_gamma,
+            weighted_captions,
+            advanced_training.save_every_n_steps,
+            advanced_training.save_last_n_steps,
+            advanced_training.save_last_n_steps_state,
+            advanced_training.use_wandb,
+            advanced_training.wandb_api_key,
+            advanced_training.scale_v_pred_loss_like_noise_pred,
+            sdxl_params.sdxl_cache_text_encoder_outputs,
+            sdxl_params.sdxl_no_half_vae,
+            advanced_training.min_timestep,
+            advanced_training.max_timestep,
+        ]
+
+        button_run.click(
+            train_model,
+            inputs=[dummy_headless] + [dummy_db_false] + settings_list,
+            show_progress=False,
+        )
+
+        button_print.click(
+            train_model,
+            inputs=[dummy_headless] + [dummy_db_true] + settings_list,
+            show_progress=False,
+        )
+
+        config.button_open_config.click(
+            open_configuration,
+            inputs=[dummy_db_true, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name] + settings_list,
+            show_progress=False,
+        )
+
+        config.button_load_config.click(
+            open_configuration,
+            inputs=[dummy_db_false, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name] + settings_list,
+            show_progress=False,
+        )
+
+        config.button_save_config.click(
+            save_configuration,
+            inputs=[dummy_db_false, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name],
+            show_progress=False,
+        )
+
+        config.button_save_as_config.click(
+            save_configuration,
+            inputs=[dummy_db_true, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name],
+            show_progress=False,
+        )
+        
+    with gr.Tab('Guides'):
+        gr.Markdown(
+            'This section provide Various Finetuning guides and information...'
+        )
+        top_level_path = './docs/Finetuning/top_level.md'
+        if os.path.exists(top_level_path):
+            with open(os.path.join(top_level_path), 'r', encoding='utf8') as file:
+                guides_top_level = file.read() + '\n'
+        gr.Markdown(guides_top_level)
+
+
+def UI(**kwargs):
+    css = ''
+
+    headless = kwargs.get('headless', False)
+    log.info(f'headless: {headless}')
+
+    if os.path.exists('./style.css'):
+        with open(os.path.join('./style.css'), 'r', encoding='utf8') as file:
+            log.info('Load CSS...')
+            css += file.read() + '\n'
+
+    interface = gr.Blocks(
+        css=css, title='Kohya_ss GUI', theme=gr.themes.Default()
+    )
+
+    with interface:
+        with gr.Tab('Finetune'):
+            finetune_tab(headless=headless)
+        with gr.Tab('Utilities'):
+            utilities_tab(enable_dreambooth_tab=False, headless=headless)
+
+    # Show the interface
+    launch_kwargs = {}
+    username = kwargs.get('username')
+    password = kwargs.get('password')
+    server_port = kwargs.get('server_port', 0)
+    inbrowser = kwargs.get('inbrowser', False)
+    share = kwargs.get('share', False)
+    server_name = kwargs.get('listen')
+
+    launch_kwargs['server_name'] = server_name
+    if username and password:
+        launch_kwargs['auth'] = (username, password)
+    if server_port > 0:
+        launch_kwargs['server_port'] = server_port
+    if inbrowser:
+        launch_kwargs['inbrowser'] = inbrowser
+    if share:
+        launch_kwargs['share'] = share
+    interface.launch(**launch_kwargs)
+
+
+if __name__ == '__main__':
+    # torch.cuda.set_per_process_memory_fraction(0.48)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--listen',
+        type=str,
+        default='127.0.0.1',
+        help='IP to listen on for connections to Gradio',
+    )
+    parser.add_argument(
+        '--username', type=str, default='', help='Username for authentication'
+    )
+    parser.add_argument(
+        '--password', type=str, default='', help='Password for authentication'
+    )
+    parser.add_argument(
+        '--server_port',
+        type=int,
+        default=0,
+        help='Port to run the server listener on',
+    )
+    parser.add_argument(
+        '--inbrowser', action='store_true', help='Open in browser'
+    )
+    parser.add_argument(
+        '--share', action='store_true', help='Share the gradio UI'
+    )
+    parser.add_argument(
+        '--headless', action='store_true', help='Is the server headless'
+    )
+
+    args = parser.parse_args()
+
+    UI(
+        username=args.username,
+        password=args.password,
+        inbrowser=args.inbrowser,
+        server_port=args.server_port,
+        share=args.share,
+        listen=args.listen,
+        headless=args.headless,
+    )
diff --git a/gen_img_diffusers.py b/gen_img_diffusers.py
new file mode 100644
index 0000000000000000000000000000000000000000..fc54ac83d287933b8d7d95838e80495b69361110
--- /dev/null
+++ b/gen_img_diffusers.py
@@ -0,0 +1,3464 @@
+"""
+VGG(
+  (features): Sequential(
+    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (1): ReLU(inplace=True)
+    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (3): ReLU(inplace=True)
+    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
+    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (6): ReLU(inplace=True)
+    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (8): ReLU(inplace=True)
+    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
+    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (11): ReLU(inplace=True)
+    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (13): ReLU(inplace=True)
+    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (15): ReLU(inplace=True)
+    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
+    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (18): ReLU(inplace=True)
+    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (20): ReLU(inplace=True)
+    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (22): ReLU(inplace=True)
+    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
+    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (25): ReLU(inplace=True)
+    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (27): ReLU(inplace=True)
+    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (29): ReLU(inplace=True)
+    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
+  )
+  (avgpool): AdaptiveAvgPool2d(output_size=(7, 7))
+  (classifier): Sequential(
+    (0): Linear(in_features=25088, out_features=4096, bias=True)
+    (1): ReLU(inplace=True)
+    (2): Dropout(p=0.5, inplace=False)
+    (3): Linear(in_features=4096, out_features=4096, bias=True)
+    (4): ReLU(inplace=True)
+    (5): Dropout(p=0.5, inplace=False)
+    (6): Linear(in_features=4096, out_features=1000, bias=True)
+  )
+)
+"""
+
+import itertools
+import json
+from typing import Any, List, NamedTuple, Optional, Tuple, Union, Callable
+import glob
+import importlib
+import inspect
+import time
+import zipfile
+from diffusers.utils import deprecate
+from diffusers.configuration_utils import FrozenDict
+import argparse
+import math
+import os
+import random
+import re
+
+import diffusers
+import numpy as np
+import torch
+import torchvision
+from diffusers import (
+    AutoencoderKL,
+    DDPMScheduler,
+    EulerAncestralDiscreteScheduler,
+    DPMSolverMultistepScheduler,
+    DPMSolverSinglestepScheduler,
+    LMSDiscreteScheduler,
+    PNDMScheduler,
+    DDIMScheduler,
+    EulerDiscreteScheduler,
+    HeunDiscreteScheduler,
+    KDPM2DiscreteScheduler,
+    KDPM2AncestralDiscreteScheduler,
+    # UNet2DConditionModel,
+    StableDiffusionPipeline,
+)
+from einops import rearrange
+from tqdm import tqdm
+from torchvision import transforms
+from transformers import CLIPTextModel, CLIPTokenizer, CLIPModel, CLIPTextConfig
+import PIL
+from PIL import Image
+from PIL.PngImagePlugin import PngInfo
+
+import library.model_util as model_util
+import library.train_util as train_util
+from networks.lora import LoRANetwork
+import tools.original_control_net as original_control_net
+from tools.original_control_net import ControlNetInfo
+from library.original_unet import UNet2DConditionModel
+from library.original_unet import FlashAttentionFunction
+
+from XTI_hijack import unet_forward_XTI, downblock_forward_XTI, upblock_forward_XTI
+
+# scheduler:
+SCHEDULER_LINEAR_START = 0.00085
+SCHEDULER_LINEAR_END = 0.0120
+SCHEDULER_TIMESTEPS = 1000
+SCHEDLER_SCHEDULE = "scaled_linear"
+
+# その他の設定
+LATENT_CHANNELS = 4
+DOWNSAMPLING_FACTOR = 8
+
+# CLIP_ID_L14_336 = "openai/clip-vit-large-patch14-336"
+
+# CLIP guided SD関連
+CLIP_MODEL_PATH = "laion/CLIP-ViT-B-32-laion2B-s34B-b79K"
+FEATURE_EXTRACTOR_SIZE = (224, 224)
+FEATURE_EXTRACTOR_IMAGE_MEAN = [0.48145466, 0.4578275, 0.40821073]
+FEATURE_EXTRACTOR_IMAGE_STD = [0.26862954, 0.26130258, 0.27577711]
+
+VGG16_IMAGE_MEAN = [0.485, 0.456, 0.406]
+VGG16_IMAGE_STD = [0.229, 0.224, 0.225]
+VGG16_INPUT_RESIZE_DIV = 4
+
+# CLIP特徴量の取得時にcutoutを使うか：使う場合にはソースを書き換えてください
+NUM_CUTOUTS = 4
+USE_CUTOUTS = False
+
+# region モジュール入れ替え部
+"""
+高速化のためのモジュール入れ替え
+"""
+
+
+def replace_unet_modules(unet: diffusers.models.unet_2d_condition.UNet2DConditionModel, mem_eff_attn, xformers, sdpa):
+    if mem_eff_attn:
+        print("Enable memory efficient attention for U-Net")
+
+        # これはDiffusersのU-Netではなく自前のU-Netなので置き換えなくても良い
+        unet.set_use_memory_efficient_attention(False, True)
+    elif xformers:
+        print("Enable xformers for U-Net")
+        try:
+            import xformers.ops
+        except ImportError:
+            raise ImportError("No xformers / xformersがインストールされていないようです")
+
+        unet.set_use_memory_efficient_attention(True, False)
+    elif sdpa:
+        print("Enable SDPA for U-Net")
+        unet.set_use_memory_efficient_attention(False, False)
+        unet.set_use_sdpa(True)
+
+
+# TODO common train_util.py
+def replace_vae_modules(vae: diffusers.models.AutoencoderKL, mem_eff_attn, xformers, sdpa):
+    if mem_eff_attn:
+        replace_vae_attn_to_memory_efficient()
+    elif xformers:
+        replace_vae_attn_to_xformers()
+    elif sdpa:
+        replace_vae_attn_to_sdpa()
+
+
+def replace_vae_attn_to_memory_efficient():
+    print("VAE Attention.forward has been replaced to FlashAttention (not xformers)")
+    flash_func = FlashAttentionFunction
+
+    def forward_flash_attn(self, hidden_states, **kwargs):
+        q_bucket_size = 512
+        k_bucket_size = 1024
+
+        residual = hidden_states
+        batch, channel, height, width = hidden_states.shape
+
+        # norm
+        hidden_states = self.group_norm(hidden_states)
+
+        hidden_states = hidden_states.view(batch, channel, height * width).transpose(1, 2)
+
+        # proj to q, k, v
+        query_proj = self.to_q(hidden_states)
+        key_proj = self.to_k(hidden_states)
+        value_proj = self.to_v(hidden_states)
+
+        query_proj, key_proj, value_proj = map(
+            lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), (query_proj, key_proj, value_proj)
+        )
+
+        out = flash_func.apply(query_proj, key_proj, value_proj, None, False, q_bucket_size, k_bucket_size)
+
+        out = rearrange(out, "b h n d -> b n (h d)")
+
+        # compute next hidden_states
+        # linear proj
+        hidden_states = self.to_out[0](hidden_states)
+        # dropout
+        hidden_states = self.to_out[1](hidden_states)
+
+        hidden_states = hidden_states.transpose(-1, -2).reshape(batch, channel, height, width)
+
+        # res connect and rescale
+        hidden_states = (hidden_states + residual) / self.rescale_output_factor
+        return hidden_states
+
+    def forward_flash_attn_0_14(self, hidden_states, **kwargs):
+        if not hasattr(self, "to_q"):
+            self.to_q = self.query
+            self.to_k = self.key
+            self.to_v = self.value
+            self.to_out = [self.proj_attn, torch.nn.Identity()]
+            self.heads = self.num_heads
+        return forward_flash_attn(self, hidden_states, **kwargs)
+
+    if diffusers.__version__ < "0.15.0":
+        diffusers.models.attention.AttentionBlock.forward = forward_flash_attn_0_14
+    else:
+        diffusers.models.attention_processor.Attention.forward = forward_flash_attn
+
+
+def replace_vae_attn_to_xformers():
+    print("VAE: Attention.forward has been replaced to xformers")
+    import xformers.ops
+
+    def forward_xformers(self, hidden_states, **kwargs):
+        residual = hidden_states
+        batch, channel, height, width = hidden_states.shape
+
+        # norm
+        hidden_states = self.group_norm(hidden_states)
+
+        hidden_states = hidden_states.view(batch, channel, height * width).transpose(1, 2)
+
+        # proj to q, k, v
+        query_proj = self.to_q(hidden_states)
+        key_proj = self.to_k(hidden_states)
+        value_proj = self.to_v(hidden_states)
+
+        query_proj, key_proj, value_proj = map(
+            lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), (query_proj, key_proj, value_proj)
+        )
+
+        query_proj = query_proj.contiguous()
+        key_proj = key_proj.contiguous()
+        value_proj = value_proj.contiguous()
+        out = xformers.ops.memory_efficient_attention(query_proj, key_proj, value_proj, attn_bias=None)
+
+        out = rearrange(out, "b h n d -> b n (h d)")
+
+        # compute next hidden_states
+        # linear proj
+        hidden_states = self.to_out[0](hidden_states)
+        # dropout
+        hidden_states = self.to_out[1](hidden_states)
+
+        hidden_states = hidden_states.transpose(-1, -2).reshape(batch, channel, height, width)
+
+        # res connect and rescale
+        hidden_states = (hidden_states + residual) / self.rescale_output_factor
+        return hidden_states
+
+    def forward_xformers_0_14(self, hidden_states, **kwargs):
+        if not hasattr(self, "to_q"):
+            self.to_q = self.query
+            self.to_k = self.key
+            self.to_v = self.value
+            self.to_out = [self.proj_attn, torch.nn.Identity()]
+            self.heads = self.num_heads
+        return forward_xformers(self, hidden_states, **kwargs)
+
+    if diffusers.__version__ < "0.15.0":
+        diffusers.models.attention.AttentionBlock.forward = forward_xformers_0_14
+    else:
+        diffusers.models.attention_processor.Attention.forward = forward_xformers
+
+
+def replace_vae_attn_to_sdpa():
+    print("VAE: Attention.forward has been replaced to sdpa")
+
+    def forward_sdpa(self, hidden_states, **kwargs):
+        residual = hidden_states
+        batch, channel, height, width = hidden_states.shape
+
+        # norm
+        hidden_states = self.group_norm(hidden_states)
+
+        hidden_states = hidden_states.view(batch, channel, height * width).transpose(1, 2)
+
+        # proj to q, k, v
+        query_proj = self.to_q(hidden_states)
+        key_proj = self.to_k(hidden_states)
+        value_proj = self.to_v(hidden_states)
+
+        query_proj, key_proj, value_proj = map(
+            lambda t: rearrange(t, "b n (h d) -> b n h d", h=self.heads), (query_proj, key_proj, value_proj)
+        )
+
+        out = torch.nn.functional.scaled_dot_product_attention(
+            query_proj, key_proj, value_proj, attn_mask=None, dropout_p=0.0, is_causal=False
+        )
+
+        out = rearrange(out, "b n h d -> b n (h d)")
+
+        # compute next hidden_states
+        # linear proj
+        hidden_states = self.to_out[0](hidden_states)
+        # dropout
+        hidden_states = self.to_out[1](hidden_states)
+
+        hidden_states = hidden_states.transpose(-1, -2).reshape(batch, channel, height, width)
+
+        # res connect and rescale
+        hidden_states = (hidden_states + residual) / self.rescale_output_factor
+        return hidden_states
+
+    def forward_sdpa_0_14(self, hidden_states, **kwargs):
+        if not hasattr(self, "to_q"):
+            self.to_q = self.query
+            self.to_k = self.key
+            self.to_v = self.value
+            self.to_out = [self.proj_attn, torch.nn.Identity()]
+            self.heads = self.num_heads
+        return forward_sdpa(self, hidden_states, **kwargs)
+
+    if diffusers.__version__ < "0.15.0":
+        diffusers.models.attention.AttentionBlock.forward = forward_sdpa_0_14
+    else:
+        diffusers.models.attention_processor.Attention.forward = forward_sdpa
+
+
+# endregion
+
+# region 画像生成の本体：lpw_stable_diffusion.py （ASL）からコピーして修正
+# https://github.com/huggingface/diffusers/blob/main/examples/community/lpw_stable_diffusion.py
+# Pipelineだけ独立して使えないのと機能追加するのとでコピーして修正
+
+
+class PipelineLike:
+    r"""
+    Pipeline for text-to-image generation using Stable Diffusion without tokens length limit, and support parsing
+    weighting in prompt.
+    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
+    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
+    Args:
+        vae ([`AutoencoderKL`]):
+            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
+        text_encoder ([`CLIPTextModel`]):
+            Frozen text-encoder. Stable Diffusion uses the text portion of
+            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
+            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
+        tokenizer (`CLIPTokenizer`):
+            Tokenizer of class
+            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
+        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
+        scheduler ([`SchedulerMixin`]):
+            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
+            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
+        safety_checker ([`StableDiffusionSafetyChecker`]):
+            Classification module that estimates whether generated images could be considered offensive or harmful.
+            Please, refer to the [model card](https://huggingface.co/CompVis/stable-diffusion-v1-4) for details.
+        feature_extractor ([`CLIPFeatureExtractor`]):
+            Model that extracts features from generated images to be used as inputs for the `safety_checker`.
+    """
+
+    def __init__(
+        self,
+        device,
+        vae: AutoencoderKL,
+        text_encoder: CLIPTextModel,
+        tokenizer: CLIPTokenizer,
+        unet: UNet2DConditionModel,
+        scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler],
+        clip_skip: int,
+        clip_model: CLIPModel,
+        clip_guidance_scale: float,
+        clip_image_guidance_scale: float,
+        vgg16_model: torchvision.models.VGG,
+        vgg16_guidance_scale: float,
+        vgg16_layer_no: int,
+        # safety_checker: StableDiffusionSafetyChecker,
+        # feature_extractor: CLIPFeatureExtractor,
+    ):
+        super().__init__()
+        self.device = device
+        self.clip_skip = clip_skip
+
+        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
+            deprecation_message = (
+                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
+                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
+                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"
+                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"
+                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"
+                " file"
+            )
+            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)
+            new_config = dict(scheduler.config)
+            new_config["steps_offset"] = 1
+            scheduler._internal_dict = FrozenDict(new_config)
+
+        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:
+            deprecation_message = (
+                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."
+                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"
+                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"
+                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"
+                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"
+            )
+            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)
+            new_config = dict(scheduler.config)
+            new_config["clip_sample"] = False
+            scheduler._internal_dict = FrozenDict(new_config)
+
+        self.vae = vae
+        self.text_encoder = text_encoder
+        self.tokenizer = tokenizer
+        self.unet = unet
+        self.scheduler = scheduler
+        self.safety_checker = None
+
+        # Textual Inversion
+        self.token_replacements = {}
+
+        # XTI
+        self.token_replacements_XTI = {}
+
+        # CLIP guidance
+        self.clip_guidance_scale = clip_guidance_scale
+        self.clip_image_guidance_scale = clip_image_guidance_scale
+        self.clip_model = clip_model
+        self.normalize = transforms.Normalize(mean=FEATURE_EXTRACTOR_IMAGE_MEAN, std=FEATURE_EXTRACTOR_IMAGE_STD)
+        self.make_cutouts = MakeCutouts(FEATURE_EXTRACTOR_SIZE)
+
+        # VGG16 guidance
+        self.vgg16_guidance_scale = vgg16_guidance_scale
+        if self.vgg16_guidance_scale > 0.0:
+            return_layers = {f"{vgg16_layer_no}": "feat"}
+            self.vgg16_feat_model = torchvision.models._utils.IntermediateLayerGetter(
+                vgg16_model.features, return_layers=return_layers
+            )
+            self.vgg16_normalize = transforms.Normalize(mean=VGG16_IMAGE_MEAN, std=VGG16_IMAGE_STD)
+
+        # ControlNet
+        self.control_nets: List[ControlNetInfo] = []
+        self.control_net_enabled = True  # control_netsが空ならTrueでもFalseでもControlNetは動作しない
+
+    # Textual Inversion
+    def add_token_replacement(self, target_token_id, rep_token_ids):
+        self.token_replacements[target_token_id] = rep_token_ids
+
+    def set_enable_control_net(self, en: bool):
+        self.control_net_enabled = en
+
+    def replace_token(self, tokens, layer=None):
+        new_tokens = []
+        for token in tokens:
+            if token in self.token_replacements:
+                replacer_ = self.token_replacements[token]
+                if layer:
+                    replacer = []
+                for r in replacer_:
+                    if r in self.token_replacements_XTI:
+                        replacer.append(self.token_replacements_XTI[r][layer])
+                    else:
+                        replacer = replacer_
+                new_tokens.extend(replacer)
+            else:
+                new_tokens.append(token)
+        return new_tokens
+
+    def add_token_replacement_XTI(self, target_token_id, rep_token_ids):
+        self.token_replacements_XTI[target_token_id] = rep_token_ids
+
+    def set_control_nets(self, ctrl_nets):
+        self.control_nets = ctrl_nets
+
+    # region xformersとか使う部分：独自に書き換えるので関係なし
+
+    def enable_xformers_memory_efficient_attention(self):
+        r"""
+        Enable memory efficient attention as implemented in xformers.
+        When this option is enabled, you should observe lower GPU memory usage and a potential speed up at inference
+        time. Speed up at training time is not guaranteed.
+        Warning: When Memory Efficient Attention and Sliced attention are both enabled, the Memory Efficient Attention
+        is used.
+        """
+        self.unet.set_use_memory_efficient_attention_xformers(True)
+
+    def disable_xformers_memory_efficient_attention(self):
+        r"""
+        Disable memory efficient attention as implemented in xformers.
+        """
+        self.unet.set_use_memory_efficient_attention_xformers(False)
+
+    def enable_attention_slicing(self, slice_size: Optional[Union[str, int]] = "auto"):
+        r"""
+        Enable sliced attention computation.
+        When this option is enabled, the attention module will split the input tensor in slices, to compute attention
+        in several steps. This is useful to save some memory in exchange for a small speed decrease.
+        Args:
+            slice_size (`str` or `int`, *optional*, defaults to `"auto"`):
+                When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If
+                a number is provided, uses as many slices as `attention_head_dim // slice_size`. In this case,
+                `attention_head_dim` must be a multiple of `slice_size`.
+        """
+        if slice_size == "auto":
+            # half the attention head size is usually a good trade-off between
+            # speed and memory
+            slice_size = self.unet.config.attention_head_dim // 2
+        self.unet.set_attention_slice(slice_size)
+
+    def disable_attention_slicing(self):
+        r"""
+        Disable sliced attention computation. If `enable_attention_slicing` was previously invoked, this method will go
+        back to computing attention in one step.
+        """
+        # set slice_size = `None` to disable `attention slicing`
+        self.enable_attention_slicing(None)
+
+    def enable_sequential_cpu_offload(self):
+        r"""
+        Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet,
+        text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a
+        `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called.
+        """
+        # accelerateが必要になるのでとりあえず省略
+        raise NotImplementedError("cpu_offload is omitted.")
+        # if is_accelerate_available():
+        #   from accelerate import cpu_offload
+        # else:
+        #   raise ImportError("Please install accelerate via `pip install accelerate`")
+
+        # device = self.device
+
+        # for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae, self.safety_checker]:
+        #   if cpu_offloaded_model is not None:
+        #     cpu_offload(cpu_offloaded_model, device)
+
+    # endregion
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        init_image: Union[torch.FloatTensor, PIL.Image.Image, List[PIL.Image.Image]] = None,
+        mask_image: Union[torch.FloatTensor, PIL.Image.Image, List[PIL.Image.Image]] = None,
+        height: int = 512,
+        width: int = 512,
+        num_inference_steps: int = 50,
+        guidance_scale: float = 7.5,
+        negative_scale: float = None,
+        strength: float = 0.8,
+        # num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[torch.Generator] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        vae_batch_size: float = None,
+        return_latents: bool = False,
+        # return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: Optional[int] = 1,
+        img2img_noise=None,
+        clip_prompts=None,
+        clip_guide_images=None,
+        networks: Optional[List[LoRANetwork]] = None,
+        **kwargs,
+    ):
+        r"""
+        Function invoked when calling the pipeline for generation.
+        Args:
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            init_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process.
+            mask_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, to mask `init_image`. White pixels in the mask will be
+                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a
+                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should
+                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.
+            height (`int`, *optional*, defaults to 512):
+                The height in pixels of the generated image.
+            width (`int`, *optional*, defaults to 512):
+                The width in pixels of the generated image.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to transform the reference `init_image`. Must be between 0 and 1.
+                `init_image` will be used as a starting point, adding more noise to it the larger the `strength`. The
+                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added
+                noise will be maximum and the denoising process will run for the full number of iterations specified in
+                `num_inference_steps`. A value of 1, therefore, essentially ignores `init_image`.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            is_cancelled_callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. If the function returns
+                `True`, the inference will be cancelled.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            `None` if cancelled by `is_cancelled_callback`,
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        num_images_per_prompt = 1  # fixed
+
+        if isinstance(prompt, str):
+            batch_size = 1
+            prompt = [prompt]
+        elif isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
+        reginonal_network = " AND " in prompt[0]
+
+        vae_batch_size = (
+            batch_size
+            if vae_batch_size is None
+            else (int(vae_batch_size) if vae_batch_size >= 1 else max(1, int(batch_size * vae_batch_size)))
+        )
+
+        if strength < 0 or strength > 1:
+            raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
+
+        if height % 8 != 0 or width % 8 != 0:
+            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
+
+        if (callback_steps is None) or (
+            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
+        ):
+            raise ValueError(
+                f"`callback_steps` has to be a positive integer but is {callback_steps} of type" f" {type(callback_steps)}."
+            )
+
+        # get prompt text embeddings
+
+        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
+        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
+        # corresponds to doing no classifier free guidance.
+        do_classifier_free_guidance = guidance_scale > 1.0
+
+        if not do_classifier_free_guidance and negative_scale is not None:
+            print(f"negative_scale is ignored if guidance scalle <= 1.0")
+            negative_scale = None
+
+        # get unconditional embeddings for classifier free guidance
+        if negative_prompt is None:
+            negative_prompt = [""] * batch_size
+        elif isinstance(negative_prompt, str):
+            negative_prompt = [negative_prompt] * batch_size
+        if batch_size != len(negative_prompt):
+            raise ValueError(
+                f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
+                f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
+                " the batch size of `prompt`."
+            )
+
+        if not self.token_replacements_XTI:
+            text_embeddings, uncond_embeddings, prompt_tokens = get_weighted_text_embeddings(
+                pipe=self,
+                prompt=prompt,
+                uncond_prompt=negative_prompt if do_classifier_free_guidance else None,
+                max_embeddings_multiples=max_embeddings_multiples,
+                clip_skip=self.clip_skip,
+                **kwargs,
+            )
+
+        if negative_scale is not None:
+            _, real_uncond_embeddings, _ = get_weighted_text_embeddings(
+                pipe=self,
+                prompt=prompt,  # こちらのトークン長に合わせてuncondを作るので75トークン超で必須
+                uncond_prompt=[""] * batch_size,
+                max_embeddings_multiples=max_embeddings_multiples,
+                clip_skip=self.clip_skip,
+                **kwargs,
+            )
+
+        if self.token_replacements_XTI:
+            text_embeddings_concat = []
+            for layer in [
+                "IN01",
+                "IN02",
+                "IN04",
+                "IN05",
+                "IN07",
+                "IN08",
+                "MID",
+                "OUT03",
+                "OUT04",
+                "OUT05",
+                "OUT06",
+                "OUT07",
+                "OUT08",
+                "OUT09",
+                "OUT10",
+                "OUT11",
+            ]:
+                text_embeddings, uncond_embeddings, prompt_tokens = get_weighted_text_embeddings(
+                    pipe=self,
+                    prompt=prompt,
+                    uncond_prompt=negative_prompt if do_classifier_free_guidance else None,
+                    max_embeddings_multiples=max_embeddings_multiples,
+                    clip_skip=self.clip_skip,
+                    layer=layer,
+                    **kwargs,
+                )
+                if do_classifier_free_guidance:
+                    if negative_scale is None:
+                        text_embeddings_concat.append(torch.cat([uncond_embeddings, text_embeddings]))
+                    else:
+                        text_embeddings_concat.append(torch.cat([uncond_embeddings, text_embeddings, real_uncond_embeddings]))
+                text_embeddings = torch.stack(text_embeddings_concat)
+        else:
+            if do_classifier_free_guidance:
+                if negative_scale is None:
+                    text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
+                else:
+                    text_embeddings = torch.cat([uncond_embeddings, text_embeddings, real_uncond_embeddings])
+
+        # CLIP guidanceで使用するembeddingsを取得する
+        if self.clip_guidance_scale > 0:
+            clip_text_input = prompt_tokens
+            if clip_text_input.shape[1] > self.tokenizer.model_max_length:
+                # TODO 75文字を超えたら警告を出す？
+                print("trim text input", clip_text_input.shape)
+                clip_text_input = torch.cat(
+                    [clip_text_input[:, : self.tokenizer.model_max_length - 1], clip_text_input[:, -1].unsqueeze(1)], dim=1
+                )
+                print("trimmed", clip_text_input.shape)
+
+            for i, clip_prompt in enumerate(clip_prompts):
+                if clip_prompt is not None:  # clip_promptがあれば上書きする
+                    clip_text_input[i] = self.tokenizer(
+                        clip_prompt,
+                        padding="max_length",
+                        max_length=self.tokenizer.model_max_length,
+                        truncation=True,
+                        return_tensors="pt",
+                    ).input_ids.to(self.device)
+
+            text_embeddings_clip = self.clip_model.get_text_features(clip_text_input)
+            text_embeddings_clip = text_embeddings_clip / text_embeddings_clip.norm(p=2, dim=-1, keepdim=True)  # prompt複数件でもOK
+
+        if (
+            self.clip_image_guidance_scale > 0
+            or self.vgg16_guidance_scale > 0
+            and clip_guide_images is not None
+            or self.control_nets
+        ):
+            if isinstance(clip_guide_images, PIL.Image.Image):
+                clip_guide_images = [clip_guide_images]
+
+            if self.clip_image_guidance_scale > 0:
+                clip_guide_images = [preprocess_guide_image(im) for im in clip_guide_images]
+                clip_guide_images = torch.cat(clip_guide_images, dim=0)
+
+                clip_guide_images = self.normalize(clip_guide_images).to(self.device).to(text_embeddings.dtype)
+                image_embeddings_clip = self.clip_model.get_image_features(clip_guide_images)
+                image_embeddings_clip = image_embeddings_clip / image_embeddings_clip.norm(p=2, dim=-1, keepdim=True)
+                if len(image_embeddings_clip) == 1:
+                    image_embeddings_clip = image_embeddings_clip.repeat((batch_size, 1, 1, 1))
+            elif self.vgg16_guidance_scale > 0:
+                size = (width // VGG16_INPUT_RESIZE_DIV, height // VGG16_INPUT_RESIZE_DIV)  # とりあえず1/4に（小さいか?）
+                clip_guide_images = [preprocess_vgg16_guide_image(im, size) for im in clip_guide_images]
+                clip_guide_images = torch.cat(clip_guide_images, dim=0)
+
+                clip_guide_images = self.vgg16_normalize(clip_guide_images).to(self.device).to(text_embeddings.dtype)
+                image_embeddings_vgg16 = self.vgg16_feat_model(clip_guide_images)["feat"]
+                if len(image_embeddings_vgg16) == 1:
+                    image_embeddings_vgg16 = image_embeddings_vgg16.repeat((batch_size, 1, 1, 1))
+            else:
+                # ControlNetのhintにguide imageを流用する
+                # 前処理はControlNet側で行う
+                pass
+
+        # set timesteps
+        self.scheduler.set_timesteps(num_inference_steps, self.device)
+
+        latents_dtype = text_embeddings.dtype
+        init_latents_orig = None
+        mask = None
+
+        if init_image is None:
+            # get the initial random noise unless the user supplied it
+
+            # Unlike in other pipelines, latents need to be generated in the target device
+            # for 1-to-1 results reproducibility with the CompVis implementation.
+            # However this currently doesn't work in `mps`.
+            latents_shape = (
+                batch_size * num_images_per_prompt,
+                self.unet.in_channels,
+                height // 8,
+                width // 8,
+            )
+
+            if latents is None:
+                if self.device.type == "mps":
+                    # randn does not exist on mps
+                    latents = torch.randn(
+                        latents_shape,
+                        generator=generator,
+                        device="cpu",
+                        dtype=latents_dtype,
+                    ).to(self.device)
+                else:
+                    latents = torch.randn(
+                        latents_shape,
+                        generator=generator,
+                        device=self.device,
+                        dtype=latents_dtype,
+                    )
+            else:
+                if latents.shape != latents_shape:
+                    raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}")
+                latents = latents.to(self.device)
+
+            timesteps = self.scheduler.timesteps.to(self.device)
+
+            # scale the initial noise by the standard deviation required by the scheduler
+            latents = latents * self.scheduler.init_noise_sigma
+        else:
+            # image to tensor
+            if isinstance(init_image, PIL.Image.Image):
+                init_image = [init_image]
+            if isinstance(init_image[0], PIL.Image.Image):
+                init_image = [preprocess_image(im) for im in init_image]
+                init_image = torch.cat(init_image)
+            if isinstance(init_image, list):
+                init_image = torch.stack(init_image)
+
+            # mask image to tensor
+            if mask_image is not None:
+                if isinstance(mask_image, PIL.Image.Image):
+                    mask_image = [mask_image]
+                if isinstance(mask_image[0], PIL.Image.Image):
+                    mask_image = torch.cat([preprocess_mask(im) for im in mask_image])  # H*W, 0 for repaint
+
+            # encode the init image into latents and scale the latents
+            init_image = init_image.to(device=self.device, dtype=latents_dtype)
+            if init_image.size()[-2:] == (height // 8, width // 8):
+                init_latents = init_image
+            else:
+                if vae_batch_size >= batch_size:
+                    init_latent_dist = self.vae.encode(init_image).latent_dist
+                    init_latents = init_latent_dist.sample(generator=generator)
+                else:
+                    if torch.cuda.is_available():
+                        torch.cuda.empty_cache()
+                    init_latents = []
+                    for i in tqdm(range(0, min(batch_size, len(init_image)), vae_batch_size)):
+                        init_latent_dist = self.vae.encode(
+                            init_image[i : i + vae_batch_size] if vae_batch_size > 1 else init_image[i].unsqueeze(0)
+                        ).latent_dist
+                        init_latents.append(init_latent_dist.sample(generator=generator))
+                    init_latents = torch.cat(init_latents)
+
+                init_latents = 0.18215 * init_latents
+
+            if len(init_latents) == 1:
+                init_latents = init_latents.repeat((batch_size, 1, 1, 1))
+            init_latents_orig = init_latents
+
+            # preprocess mask
+            if mask_image is not None:
+                mask = mask_image.to(device=self.device, dtype=latents_dtype)
+                if len(mask) == 1:
+                    mask = mask.repeat((batch_size, 1, 1, 1))
+
+                # check sizes
+                if not mask.shape == init_latents.shape:
+                    raise ValueError("The mask and init_image should be the same size!")
+
+            # get the original timestep using init_timestep
+            offset = self.scheduler.config.get("steps_offset", 0)
+            init_timestep = int(num_inference_steps * strength) + offset
+            init_timestep = min(init_timestep, num_inference_steps)
+
+            timesteps = self.scheduler.timesteps[-init_timestep]
+            timesteps = torch.tensor([timesteps] * batch_size * num_images_per_prompt, device=self.device)
+
+            # add noise to latents using the timesteps
+            latents = self.scheduler.add_noise(init_latents, img2img_noise, timesteps)
+
+            t_start = max(num_inference_steps - init_timestep + offset, 0)
+            timesteps = self.scheduler.timesteps[t_start:].to(self.device)
+
+        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
+        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
+        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
+        # and should be between [0, 1]
+        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
+        extra_step_kwargs = {}
+        if accepts_eta:
+            extra_step_kwargs["eta"] = eta
+
+        num_latent_input = (3 if negative_scale is not None else 2) if do_classifier_free_guidance else 1
+
+        if self.control_nets:
+            guided_hints = original_control_net.get_guided_hints(self.control_nets, num_latent_input, batch_size, clip_guide_images)
+
+        for i, t in enumerate(tqdm(timesteps)):
+            # expand the latents if we are doing classifier free guidance
+            latent_model_input = latents.repeat((num_latent_input, 1, 1, 1))
+            latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+            # predict the noise residual
+            if self.control_nets and self.control_net_enabled:
+                if reginonal_network:
+                    num_sub_and_neg_prompts = len(text_embeddings) // batch_size
+                    text_emb_last = text_embeddings[num_sub_and_neg_prompts - 2 :: num_sub_and_neg_prompts]  # last subprompt
+                else:
+                    text_emb_last = text_embeddings
+                noise_pred = original_control_net.call_unet_and_control_net(
+                    i,
+                    num_latent_input,
+                    self.unet,
+                    self.control_nets,
+                    guided_hints,
+                    i / len(timesteps),
+                    latent_model_input,
+                    t,
+                    text_emb_last,
+                ).sample
+            else:
+                noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
+
+            # perform guidance
+            if do_classifier_free_guidance:
+                if negative_scale is None:
+                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(num_latent_input)  # uncond by negative prompt
+                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+                else:
+                    noise_pred_negative, noise_pred_text, noise_pred_uncond = noise_pred.chunk(
+                        num_latent_input
+                    )  # uncond is real uncond
+                    noise_pred = (
+                        noise_pred_uncond
+                        + guidance_scale * (noise_pred_text - noise_pred_uncond)
+                        - negative_scale * (noise_pred_negative - noise_pred_uncond)
+                    )
+
+            # perform clip guidance
+            if self.clip_guidance_scale > 0 or self.clip_image_guidance_scale > 0 or self.vgg16_guidance_scale > 0:
+                text_embeddings_for_guidance = (
+                    text_embeddings.chunk(num_latent_input)[1] if do_classifier_free_guidance else text_embeddings
+                )
+
+                if self.clip_guidance_scale > 0:
+                    noise_pred, latents = self.cond_fn(
+                        latents,
+                        t,
+                        i,
+                        text_embeddings_for_guidance,
+                        noise_pred,
+                        text_embeddings_clip,
+                        self.clip_guidance_scale,
+                        NUM_CUTOUTS,
+                        USE_CUTOUTS,
+                    )
+                if self.clip_image_guidance_scale > 0 and clip_guide_images is not None:
+                    noise_pred, latents = self.cond_fn(
+                        latents,
+                        t,
+                        i,
+                        text_embeddings_for_guidance,
+                        noise_pred,
+                        image_embeddings_clip,
+                        self.clip_image_guidance_scale,
+                        NUM_CUTOUTS,
+                        USE_CUTOUTS,
+                    )
+                if self.vgg16_guidance_scale > 0 and clip_guide_images is not None:
+                    noise_pred, latents = self.cond_fn_vgg16(
+                        latents, t, i, text_embeddings_for_guidance, noise_pred, image_embeddings_vgg16, self.vgg16_guidance_scale
+                    )
+
+            # compute the previous noisy sample x_t -> x_t-1
+            latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
+
+            if mask is not None:
+                # masking
+                init_latents_proper = self.scheduler.add_noise(init_latents_orig, img2img_noise, torch.tensor([t]))
+                latents = (init_latents_proper * mask) + (latents * (1 - mask))
+
+            # call the callback, if provided
+            if i % callback_steps == 0:
+                if callback is not None:
+                    callback(i, t, latents)
+                if is_cancelled_callback is not None and is_cancelled_callback():
+                    return None
+
+        if return_latents:
+            return (latents, False)
+
+        latents = 1 / 0.18215 * latents
+        if vae_batch_size >= batch_size:
+            image = self.vae.decode(latents).sample
+        else:
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+            images = []
+            for i in tqdm(range(0, batch_size, vae_batch_size)):
+                images.append(
+                    self.vae.decode(latents[i : i + vae_batch_size] if vae_batch_size > 1 else latents[i].unsqueeze(0)).sample
+                )
+            image = torch.cat(images)
+
+        image = (image / 2 + 0.5).clamp(0, 1)
+
+        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16
+        image = image.cpu().permute(0, 2, 3, 1).float().numpy()
+
+        if self.safety_checker is not None:
+            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(self.device)
+            image, has_nsfw_concept = self.safety_checker(
+                images=image,
+                clip_input=safety_checker_input.pixel_values.to(text_embeddings.dtype),
+            )
+        else:
+            has_nsfw_concept = None
+
+        if output_type == "pil":
+            # image = self.numpy_to_pil(image)
+            image = (image * 255).round().astype("uint8")
+            image = [Image.fromarray(im) for im in image]
+
+        # if not return_dict:
+        return (image, has_nsfw_concept)
+
+        # return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)
+
+    def text2img(
+        self,
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        height: int = 512,
+        width: int = 512,
+        num_inference_steps: int = 50,
+        guidance_scale: float = 7.5,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[torch.Generator] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        callback_steps: Optional[int] = 1,
+        **kwargs,
+    ):
+        r"""
+        Function for text-to-image generation.
+        Args:
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            height (`int`, *optional*, defaults to 512):
+                The height in pixels of the generated image.
+            width (`int`, *optional*, defaults to 512):
+                The width in pixels of the generated image.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        return self.__call__(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            height=height,
+            width=width,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            num_images_per_prompt=num_images_per_prompt,
+            eta=eta,
+            generator=generator,
+            latents=latents,
+            max_embeddings_multiples=max_embeddings_multiples,
+            output_type=output_type,
+            return_dict=return_dict,
+            callback=callback,
+            callback_steps=callback_steps,
+            **kwargs,
+        )
+
+    def img2img(
+        self,
+        init_image: Union[torch.FloatTensor, PIL.Image.Image],
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        strength: float = 0.8,
+        num_inference_steps: Optional[int] = 50,
+        guidance_scale: Optional[float] = 7.5,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: Optional[float] = 0.0,
+        generator: Optional[torch.Generator] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        callback_steps: Optional[int] = 1,
+        **kwargs,
+    ):
+        r"""
+        Function for image-to-image generation.
+        Args:
+            init_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process.
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to transform the reference `init_image`. Must be between 0 and 1.
+                `init_image` will be used as a starting point, adding more noise to it the larger the `strength`. The
+                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added
+                noise will be maximum and the denoising process will run for the full number of iterations specified in
+                `num_inference_steps`. A value of 1, therefore, essentially ignores `init_image`.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference. This parameter will be modulated by `strength`.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        return self.__call__(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            init_image=init_image,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            strength=strength,
+            num_images_per_prompt=num_images_per_prompt,
+            eta=eta,
+            generator=generator,
+            max_embeddings_multiples=max_embeddings_multiples,
+            output_type=output_type,
+            return_dict=return_dict,
+            callback=callback,
+            callback_steps=callback_steps,
+            **kwargs,
+        )
+
+    def inpaint(
+        self,
+        init_image: Union[torch.FloatTensor, PIL.Image.Image],
+        mask_image: Union[torch.FloatTensor, PIL.Image.Image],
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        strength: float = 0.8,
+        num_inference_steps: Optional[int] = 50,
+        guidance_scale: Optional[float] = 7.5,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: Optional[float] = 0.0,
+        generator: Optional[torch.Generator] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        callback_steps: Optional[int] = 1,
+        **kwargs,
+    ):
+        r"""
+        Function for inpaint.
+        Args:
+            init_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process. This is the image whose masked region will be inpainted.
+            mask_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, to mask `init_image`. White pixels in the mask will be
+                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a
+                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should
+                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to inpaint the masked area. Must be between 0 and 1. When `strength`
+                is 1, the denoising process will be run on the masked area for the full number of iterations specified
+                in `num_inference_steps`. `init_image` will be used as a reference for the masked area, adding more
+                noise to that region the larger the `strength`. If `strength` is 0, no inpainting will occur.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The reference number of denoising steps. More denoising steps usually lead to a higher quality image at
+                the expense of slower inference. This parameter will be modulated by `strength`, as explained above.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        return self.__call__(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            init_image=init_image,
+            mask_image=mask_image,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            strength=strength,
+            num_images_per_prompt=num_images_per_prompt,
+            eta=eta,
+            generator=generator,
+            max_embeddings_multiples=max_embeddings_multiples,
+            output_type=output_type,
+            return_dict=return_dict,
+            callback=callback,
+            callback_steps=callback_steps,
+            **kwargs,
+        )
+
+    # CLIP guidance StableDiffusion
+    # copy from https://github.com/huggingface/diffusers/blob/main/examples/community/clip_guided_stable_diffusion.py
+
+    # バッチを分解して1件ずつ処理する
+    def cond_fn(
+        self,
+        latents,
+        timestep,
+        index,
+        text_embeddings,
+        noise_pred_original,
+        guide_embeddings_clip,
+        clip_guidance_scale,
+        num_cutouts,
+        use_cutouts=True,
+    ):
+        if len(latents) == 1:
+            return self.cond_fn1(
+                latents,
+                timestep,
+                index,
+                text_embeddings,
+                noise_pred_original,
+                guide_embeddings_clip,
+                clip_guidance_scale,
+                num_cutouts,
+                use_cutouts,
+            )
+
+        noise_pred = []
+        cond_latents = []
+        for i in range(len(latents)):
+            lat1 = latents[i].unsqueeze(0)
+            tem1 = text_embeddings[i].unsqueeze(0)
+            npo1 = noise_pred_original[i].unsqueeze(0)
+            gem1 = guide_embeddings_clip[i].unsqueeze(0)
+            npr1, cla1 = self.cond_fn1(lat1, timestep, index, tem1, npo1, gem1, clip_guidance_scale, num_cutouts, use_cutouts)
+            noise_pred.append(npr1)
+            cond_latents.append(cla1)
+
+        noise_pred = torch.cat(noise_pred)
+        cond_latents = torch.cat(cond_latents)
+        return noise_pred, cond_latents
+
+    @torch.enable_grad()
+    def cond_fn1(
+        self,
+        latents,
+        timestep,
+        index,
+        text_embeddings,
+        noise_pred_original,
+        guide_embeddings_clip,
+        clip_guidance_scale,
+        num_cutouts,
+        use_cutouts=True,
+    ):
+        latents = latents.detach().requires_grad_()
+
+        if isinstance(self.scheduler, LMSDiscreteScheduler):
+            sigma = self.scheduler.sigmas[index]
+            # the model input needs to be scaled to match the continuous ODE formulation in K-LMS
+            latent_model_input = latents / ((sigma**2 + 1) ** 0.5)
+        else:
+            latent_model_input = latents
+
+        # predict the noise residual
+        noise_pred = self.unet(latent_model_input, timestep, encoder_hidden_states=text_embeddings).sample
+
+        if isinstance(self.scheduler, (PNDMScheduler, DDIMScheduler)):
+            alpha_prod_t = self.scheduler.alphas_cumprod[timestep]
+            beta_prod_t = 1 - alpha_prod_t
+            # compute predicted original sample from predicted noise also called
+            # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
+            pred_original_sample = (latents - beta_prod_t ** (0.5) * noise_pred) / alpha_prod_t ** (0.5)
+
+            fac = torch.sqrt(beta_prod_t)
+            sample = pred_original_sample * (fac) + latents * (1 - fac)
+        elif isinstance(self.scheduler, LMSDiscreteScheduler):
+            sigma = self.scheduler.sigmas[index]
+            sample = latents - sigma * noise_pred
+        else:
+            raise ValueError(f"scheduler type {type(self.scheduler)} not supported")
+
+        sample = 1 / 0.18215 * sample
+        image = self.vae.decode(sample).sample
+        image = (image / 2 + 0.5).clamp(0, 1)
+
+        if use_cutouts:
+            image = self.make_cutouts(image, num_cutouts)
+        else:
+            image = transforms.Resize(FEATURE_EXTRACTOR_SIZE)(image)
+        image = self.normalize(image).to(latents.dtype)
+
+        image_embeddings_clip = self.clip_model.get_image_features(image)
+        image_embeddings_clip = image_embeddings_clip / image_embeddings_clip.norm(p=2, dim=-1, keepdim=True)
+
+        if use_cutouts:
+            dists = spherical_dist_loss(image_embeddings_clip, guide_embeddings_clip)
+            dists = dists.view([num_cutouts, sample.shape[0], -1])
+            loss = dists.sum(2).mean(0).sum() * clip_guidance_scale
+        else:
+            # バッチサイズが複数だと正しく動くかわからない
+            loss = spherical_dist_loss(image_embeddings_clip, guide_embeddings_clip).mean() * clip_guidance_scale
+
+        grads = -torch.autograd.grad(loss, latents)[0]
+
+        if isinstance(self.scheduler, LMSDiscreteScheduler):
+            latents = latents.detach() + grads * (sigma**2)
+            noise_pred = noise_pred_original
+        else:
+            noise_pred = noise_pred_original - torch.sqrt(beta_prod_t) * grads
+        return noise_pred, latents
+
+    # バッチを分解して一件ずつ処理する
+    def cond_fn_vgg16(self, latents, timestep, index, text_embeddings, noise_pred_original, guide_embeddings, guidance_scale):
+        if len(latents) == 1:
+            return self.cond_fn_vgg16_b1(
+                latents, timestep, index, text_embeddings, noise_pred_original, guide_embeddings, guidance_scale
+            )
+
+        noise_pred = []
+        cond_latents = []
+        for i in range(len(latents)):
+            lat1 = latents[i].unsqueeze(0)
+            tem1 = text_embeddings[i].unsqueeze(0)
+            npo1 = noise_pred_original[i].unsqueeze(0)
+            gem1 = guide_embeddings[i].unsqueeze(0)
+            npr1, cla1 = self.cond_fn_vgg16_b1(lat1, timestep, index, tem1, npo1, gem1, guidance_scale)
+            noise_pred.append(npr1)
+            cond_latents.append(cla1)
+
+        noise_pred = torch.cat(noise_pred)
+        cond_latents = torch.cat(cond_latents)
+        return noise_pred, cond_latents
+
+    # 1件だけ処理する
+    @torch.enable_grad()
+    def cond_fn_vgg16_b1(self, latents, timestep, index, text_embeddings, noise_pred_original, guide_embeddings, guidance_scale):
+        latents = latents.detach().requires_grad_()
+
+        if isinstance(self.scheduler, LMSDiscreteScheduler):
+            sigma = self.scheduler.sigmas[index]
+            # the model input needs to be scaled to match the continuous ODE formulation in K-LMS
+            latent_model_input = latents / ((sigma**2 + 1) ** 0.5)
+        else:
+            latent_model_input = latents
+
+        # predict the noise residual
+        noise_pred = self.unet(latent_model_input, timestep, encoder_hidden_states=text_embeddings).sample
+
+        if isinstance(self.scheduler, (PNDMScheduler, DDIMScheduler)):
+            alpha_prod_t = self.scheduler.alphas_cumprod[timestep]
+            beta_prod_t = 1 - alpha_prod_t
+            # compute predicted original sample from predicted noise also called
+            # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
+            pred_original_sample = (latents - beta_prod_t ** (0.5) * noise_pred) / alpha_prod_t ** (0.5)
+
+            fac = torch.sqrt(beta_prod_t)
+            sample = pred_original_sample * (fac) + latents * (1 - fac)
+        elif isinstance(self.scheduler, LMSDiscreteScheduler):
+            sigma = self.scheduler.sigmas[index]
+            sample = latents - sigma * noise_pred
+        else:
+            raise ValueError(f"scheduler type {type(self.scheduler)} not supported")
+
+        sample = 1 / 0.18215 * sample
+        image = self.vae.decode(sample).sample
+        image = (image / 2 + 0.5).clamp(0, 1)
+        image = transforms.Resize((image.shape[-2] // VGG16_INPUT_RESIZE_DIV, image.shape[-1] // VGG16_INPUT_RESIZE_DIV))(image)
+        image = self.vgg16_normalize(image).to(latents.dtype)
+
+        image_embeddings = self.vgg16_feat_model(image)["feat"]
+
+        # バッチサイズが複数だと正しく動くかわからない
+        loss = ((image_embeddings - guide_embeddings) ** 2).mean() * guidance_scale  # MSE style transferでコンテンツの損失はMSEなので
+
+        grads = -torch.autograd.grad(loss, latents)[0]
+        if isinstance(self.scheduler, LMSDiscreteScheduler):
+            latents = latents.detach() + grads * (sigma**2)
+            noise_pred = noise_pred_original
+        else:
+            noise_pred = noise_pred_original - torch.sqrt(beta_prod_t) * grads
+        return noise_pred, latents
+
+
+class MakeCutouts(torch.nn.Module):
+    def __init__(self, cut_size, cut_power=1.0):
+        super().__init__()
+
+        self.cut_size = cut_size
+        self.cut_power = cut_power
+
+    def forward(self, pixel_values, num_cutouts):
+        sideY, sideX = pixel_values.shape[2:4]
+        max_size = min(sideX, sideY)
+        min_size = min(sideX, sideY, self.cut_size)
+        cutouts = []
+        for _ in range(num_cutouts):
+            size = int(torch.rand([]) ** self.cut_power * (max_size - min_size) + min_size)
+            offsetx = torch.randint(0, sideX - size + 1, ())
+            offsety = torch.randint(0, sideY - size + 1, ())
+            cutout = pixel_values[:, :, offsety : offsety + size, offsetx : offsetx + size]
+            cutouts.append(torch.nn.functional.adaptive_avg_pool2d(cutout, self.cut_size))
+        return torch.cat(cutouts)
+
+
+def spherical_dist_loss(x, y):
+    x = torch.nn.functional.normalize(x, dim=-1)
+    y = torch.nn.functional.normalize(y, dim=-1)
+    return (x - y).norm(dim=-1).div(2).arcsin().pow(2).mul(2)
+
+
+re_attention = re.compile(
+    r"""
+\\\(|
+\\\)|
+\\\[|
+\\]|
+\\\\|
+\\|
+\(|
+\[|
+:([+-]?[.\d]+)\)|
+\)|
+]|
+[^\\()\[\]:]+|
+:
+""",
+    re.X,
+)
+
+
+def parse_prompt_attention(text):
+    """
+    Parses a string with attention tokens and returns a list of pairs: text and its associated weight.
+    Accepted tokens are:
+      (abc) - increases attention to abc by a multiplier of 1.1
+      (abc:3.12) - increases attention to abc by a multiplier of 3.12
+      [abc] - decreases attention to abc by a multiplier of 1.1
+      \( - literal character '('
+      \[ - literal character '['
+      \) - literal character ')'
+      \] - literal character ']'
+      \\ - literal character '\'
+      anything else - just text
+    >>> parse_prompt_attention('normal text')
+    [['normal text', 1.0]]
+    >>> parse_prompt_attention('an (important) word')
+    [['an ', 1.0], ['important', 1.1], [' word', 1.0]]
+    >>> parse_prompt_attention('(unbalanced')
+    [['unbalanced', 1.1]]
+    >>> parse_prompt_attention('\(literal\]')
+    [['(literal]', 1.0]]
+    >>> parse_prompt_attention('(unnecessary)(parens)')
+    [['unnecessaryparens', 1.1]]
+    >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')
+    [['a ', 1.0],
+     ['house', 1.5730000000000004],
+     [' ', 1.1],
+     ['on', 1.0],
+     [' a ', 1.1],
+     ['hill', 0.55],
+     [', sun, ', 1.1],
+     ['sky', 1.4641000000000006],
+     ['.', 1.1]]
+    """
+
+    res = []
+    round_brackets = []
+    square_brackets = []
+
+    round_bracket_multiplier = 1.1
+    square_bracket_multiplier = 1 / 1.1
+
+    def multiply_range(start_position, multiplier):
+        for p in range(start_position, len(res)):
+            res[p][1] *= multiplier
+
+    # keep break as separate token
+    text = text.replace("BREAK", "\\BREAK\\")
+
+    for m in re_attention.finditer(text):
+        text = m.group(0)
+        weight = m.group(1)
+
+        if text.startswith("\\"):
+            res.append([text[1:], 1.0])
+        elif text == "(":
+            round_brackets.append(len(res))
+        elif text == "[":
+            square_brackets.append(len(res))
+        elif weight is not None and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), float(weight))
+        elif text == ")" and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), round_bracket_multiplier)
+        elif text == "]" and len(square_brackets) > 0:
+            multiply_range(square_brackets.pop(), square_bracket_multiplier)
+        else:
+            res.append([text, 1.0])
+
+    for pos in round_brackets:
+        multiply_range(pos, round_bracket_multiplier)
+
+    for pos in square_brackets:
+        multiply_range(pos, square_bracket_multiplier)
+
+    if len(res) == 0:
+        res = [["", 1.0]]
+
+    # merge runs of identical weights
+    i = 0
+    while i + 1 < len(res):
+        if res[i][1] == res[i + 1][1] and res[i][0].strip() != "BREAK" and res[i + 1][0].strip() != "BREAK":
+            res[i][0] += res[i + 1][0]
+            res.pop(i + 1)
+        else:
+            i += 1
+
+    return res
+
+
+def get_prompts_with_weights(pipe: PipelineLike, prompt: List[str], max_length: int, layer=None):
+    r"""
+    Tokenize a list of prompts and return its tokens with weights of each token.
+    No padding, starting or ending token is included.
+    """
+    tokens = []
+    weights = []
+    truncated = False
+
+    for text in prompt:
+        texts_and_weights = parse_prompt_attention(text)
+        text_token = []
+        text_weight = []
+        for word, weight in texts_and_weights:
+            if word.strip() == "BREAK":
+                # pad until next multiple of tokenizer's max token length
+                pad_len = pipe.tokenizer.model_max_length - (len(text_token) % pipe.tokenizer.model_max_length)
+                print(f"BREAK pad_len: {pad_len}")
+                for i in range(pad_len):
+                    # v2のときEOSをつけるべきかどうかわからないぜ
+                    # if i == 0:
+                    #     text_token.append(pipe.tokenizer.eos_token_id)
+                    # else:
+                    text_token.append(pipe.tokenizer.pad_token_id)
+                    text_weight.append(1.0)
+                continue
+
+            # tokenize and discard the starting and the ending token
+            token = pipe.tokenizer(word).input_ids[1:-1]
+
+            token = pipe.replace_token(token, layer=layer)
+
+            text_token += token
+            # copy the weight by length of token
+            text_weight += [weight] * len(token)
+            # stop if the text is too long (longer than truncation limit)
+            if len(text_token) > max_length:
+                truncated = True
+                break
+        # truncate
+        if len(text_token) > max_length:
+            truncated = True
+            text_token = text_token[:max_length]
+            text_weight = text_weight[:max_length]
+        tokens.append(text_token)
+        weights.append(text_weight)
+    if truncated:
+        print("warning: Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples")
+    return tokens, weights
+
+
+def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, pad, no_boseos_middle=True, chunk_length=77):
+    r"""
+    Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length.
+    """
+    max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)
+    weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length
+    for i in range(len(tokens)):
+        tokens[i] = [bos] + tokens[i] + [eos] + [pad] * (max_length - 2 - len(tokens[i]))
+        if no_boseos_middle:
+            weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))
+        else:
+            w = []
+            if len(weights[i]) == 0:
+                w = [1.0] * weights_length
+            else:
+                for j in range(max_embeddings_multiples):
+                    w.append(1.0)  # weight for starting token in this chunk
+                    w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]
+                    w.append(1.0)  # weight for ending token in this chunk
+                w += [1.0] * (weights_length - len(w))
+            weights[i] = w[:]
+
+    return tokens, weights
+
+
+def get_unweighted_text_embeddings(
+    pipe: PipelineLike,
+    text_input: torch.Tensor,
+    chunk_length: int,
+    clip_skip: int,
+    eos: int,
+    pad: int,
+    no_boseos_middle: Optional[bool] = True,
+):
+    """
+    When the length of tokens is a multiple of the capacity of the text encoder,
+    it should be split into chunks and sent to the text encoder individually.
+    """
+    max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)
+    if max_embeddings_multiples > 1:
+        text_embeddings = []
+        for i in range(max_embeddings_multiples):
+            # extract the i-th chunk
+            text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()
+
+            # cover the head and the tail by the starting and the ending tokens
+            text_input_chunk[:, 0] = text_input[0, 0]
+            if pad == eos:  # v1
+                text_input_chunk[:, -1] = text_input[0, -1]
+            else:  # v2
+                for j in range(len(text_input_chunk)):
+                    if text_input_chunk[j, -1] != eos and text_input_chunk[j, -1] != pad:  # 最後に普通の文字がある
+                        text_input_chunk[j, -1] = eos
+                    if text_input_chunk[j, 1] == pad:  # BOSだけであとはPAD
+                        text_input_chunk[j, 1] = eos
+
+            if clip_skip is None or clip_skip == 1:
+                text_embedding = pipe.text_encoder(text_input_chunk)[0]
+            else:
+                enc_out = pipe.text_encoder(text_input_chunk, output_hidden_states=True, return_dict=True)
+                text_embedding = enc_out["hidden_states"][-clip_skip]
+                text_embedding = pipe.text_encoder.text_model.final_layer_norm(text_embedding)
+
+            if no_boseos_middle:
+                if i == 0:
+                    # discard the ending token
+                    text_embedding = text_embedding[:, :-1]
+                elif i == max_embeddings_multiples - 1:
+                    # discard the starting token
+                    text_embedding = text_embedding[:, 1:]
+                else:
+                    # discard both starting and ending tokens
+                    text_embedding = text_embedding[:, 1:-1]
+
+            text_embeddings.append(text_embedding)
+        text_embeddings = torch.concat(text_embeddings, axis=1)
+    else:
+        if clip_skip is None or clip_skip == 1:
+            text_embeddings = pipe.text_encoder(text_input)[0]
+        else:
+            enc_out = pipe.text_encoder(text_input, output_hidden_states=True, return_dict=True)
+            text_embeddings = enc_out["hidden_states"][-clip_skip]
+            text_embeddings = pipe.text_encoder.text_model.final_layer_norm(text_embeddings)
+    return text_embeddings
+
+
+def get_weighted_text_embeddings(
+    pipe: PipelineLike,
+    prompt: Union[str, List[str]],
+    uncond_prompt: Optional[Union[str, List[str]]] = None,
+    max_embeddings_multiples: Optional[int] = 1,
+    no_boseos_middle: Optional[bool] = False,
+    skip_parsing: Optional[bool] = False,
+    skip_weighting: Optional[bool] = False,
+    clip_skip=None,
+    layer=None,
+    **kwargs,
+):
+    r"""
+    Prompts can be assigned with local weights using brackets. For example,
+    prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful',
+    and the embedding tokens corresponding to the words get multiplied by a constant, 1.1.
+    Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean.
+    Args:
+        pipe (`DiffusionPipeline`):
+            Pipe to provide access to the tokenizer and the text encoder.
+        prompt (`str` or `List[str]`):
+            The prompt or prompts to guide the image generation.
+        uncond_prompt (`str` or `List[str]`):
+            The unconditional prompt or prompts for guide the image generation. If unconditional prompt
+            is provided, the embeddings of prompt and uncond_prompt are concatenated.
+        max_embeddings_multiples (`int`, *optional*, defaults to `1`):
+            The max multiple length of prompt embeddings compared to the max output length of text encoder.
+        no_boseos_middle (`bool`, *optional*, defaults to `False`):
+            If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and
+            ending token in each of the chunk in the middle.
+        skip_parsing (`bool`, *optional*, defaults to `False`):
+            Skip the parsing of brackets.
+        skip_weighting (`bool`, *optional*, defaults to `False`):
+            Skip the weighting. When the parsing is skipped, it is forced True.
+    """
+    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+    if isinstance(prompt, str):
+        prompt = [prompt]
+
+    # split the prompts with "AND". each prompt must have the same number of splits
+    new_prompts = []
+    for p in prompt:
+        new_prompts.extend(p.split(" AND "))
+    prompt = new_prompts
+
+    if not skip_parsing:
+        prompt_tokens, prompt_weights = get_prompts_with_weights(pipe, prompt, max_length - 2, layer=layer)
+        if uncond_prompt is not None:
+            if isinstance(uncond_prompt, str):
+                uncond_prompt = [uncond_prompt]
+            uncond_tokens, uncond_weights = get_prompts_with_weights(pipe, uncond_prompt, max_length - 2, layer=layer)
+    else:
+        prompt_tokens = [token[1:-1] for token in pipe.tokenizer(prompt, max_length=max_length, truncation=True).input_ids]
+        prompt_weights = [[1.0] * len(token) for token in prompt_tokens]
+        if uncond_prompt is not None:
+            if isinstance(uncond_prompt, str):
+                uncond_prompt = [uncond_prompt]
+            uncond_tokens = [
+                token[1:-1] for token in pipe.tokenizer(uncond_prompt, max_length=max_length, truncation=True).input_ids
+            ]
+            uncond_weights = [[1.0] * len(token) for token in uncond_tokens]
+
+    # round up the longest length of tokens to a multiple of (model_max_length - 2)
+    max_length = max([len(token) for token in prompt_tokens])
+    if uncond_prompt is not None:
+        max_length = max(max_length, max([len(token) for token in uncond_tokens]))
+
+    max_embeddings_multiples = min(
+        max_embeddings_multiples,
+        (max_length - 1) // (pipe.tokenizer.model_max_length - 2) + 1,
+    )
+    max_embeddings_multiples = max(1, max_embeddings_multiples)
+    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+
+    # pad the length of tokens and weights
+    bos = pipe.tokenizer.bos_token_id
+    eos = pipe.tokenizer.eos_token_id
+    pad = pipe.tokenizer.pad_token_id
+    prompt_tokens, prompt_weights = pad_tokens_and_weights(
+        prompt_tokens,
+        prompt_weights,
+        max_length,
+        bos,
+        eos,
+        pad,
+        no_boseos_middle=no_boseos_middle,
+        chunk_length=pipe.tokenizer.model_max_length,
+    )
+    prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device=pipe.device)
+    if uncond_prompt is not None:
+        uncond_tokens, uncond_weights = pad_tokens_and_weights(
+            uncond_tokens,
+            uncond_weights,
+            max_length,
+            bos,
+            eos,
+            pad,
+            no_boseos_middle=no_boseos_middle,
+            chunk_length=pipe.tokenizer.model_max_length,
+        )
+        uncond_tokens = torch.tensor(uncond_tokens, dtype=torch.long, device=pipe.device)
+
+    # get the embeddings
+    text_embeddings = get_unweighted_text_embeddings(
+        pipe,
+        prompt_tokens,
+        pipe.tokenizer.model_max_length,
+        clip_skip,
+        eos,
+        pad,
+        no_boseos_middle=no_boseos_middle,
+    )
+    prompt_weights = torch.tensor(prompt_weights, dtype=text_embeddings.dtype, device=pipe.device)
+    if uncond_prompt is not None:
+        uncond_embeddings = get_unweighted_text_embeddings(
+            pipe,
+            uncond_tokens,
+            pipe.tokenizer.model_max_length,
+            clip_skip,
+            eos,
+            pad,
+            no_boseos_middle=no_boseos_middle,
+        )
+        uncond_weights = torch.tensor(uncond_weights, dtype=uncond_embeddings.dtype, device=pipe.device)
+
+    # assign weights to the prompts and normalize in the sense of mean
+    # TODO: should we normalize by chunk or in a whole (current implementation)?
+    # →全体でいいんじゃないかな
+    if (not skip_parsing) and (not skip_weighting):
+        previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+        text_embeddings *= prompt_weights.unsqueeze(-1)
+        current_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+        text_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)
+        if uncond_prompt is not None:
+            previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)
+            uncond_embeddings *= uncond_weights.unsqueeze(-1)
+            current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)
+            uncond_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)
+
+    if uncond_prompt is not None:
+        return text_embeddings, uncond_embeddings, prompt_tokens
+    return text_embeddings, None, prompt_tokens
+
+
+def preprocess_guide_image(image):
+    image = image.resize(FEATURE_EXTRACTOR_SIZE, resample=Image.NEAREST)  # cond_fnと合わせる
+    image = np.array(image).astype(np.float32) / 255.0
+    image = image[None].transpose(0, 3, 1, 2)  # nchw
+    image = torch.from_numpy(image)
+    return image  # 0 to 1
+
+
+# VGG16の入力は任意サイズでよいので入力画像を適宜リサイズする
+def preprocess_vgg16_guide_image(image, size):
+    image = image.resize(size, resample=Image.NEAREST)  # cond_fnと合わせる
+    image = np.array(image).astype(np.float32) / 255.0
+    image = image[None].transpose(0, 3, 1, 2)  # nchw
+    image = torch.from_numpy(image)
+    return image  # 0 to 1
+
+
+def preprocess_image(image):
+    w, h = image.size
+    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32
+    image = image.resize((w, h), resample=PIL.Image.LANCZOS)
+    image = np.array(image).astype(np.float32) / 255.0
+    image = image[None].transpose(0, 3, 1, 2)
+    image = torch.from_numpy(image)
+    return 2.0 * image - 1.0
+
+
+def preprocess_mask(mask):
+    mask = mask.convert("L")
+    w, h = mask.size
+    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32
+    mask = mask.resize((w // 8, h // 8), resample=PIL.Image.BILINEAR)  # LANCZOS)
+    mask = np.array(mask).astype(np.float32) / 255.0
+    mask = np.tile(mask, (4, 1, 1))
+    mask = mask[None].transpose(0, 1, 2, 3)  # what does this step do?
+    mask = 1 - mask  # repaint white, keep black
+    mask = torch.from_numpy(mask)
+    return mask
+
+
+# regular expression for dynamic prompt:
+# starts and ends with "{" and "}"
+# contains at least one variant divided by "|"
+# optional framgments divided by "$$" at start
+# if the first fragment is "E" or "e", enumerate all variants
+# if the second fragment is a number or two numbers, repeat the variants in the range
+# if the third fragment is a string, use it as a separator
+
+RE_DYNAMIC_PROMPT = re.compile(r"\{((e|E)\$\$)?(([\d\-]+)\$\$)?(([^\|\}]+?)\$\$)?(.+?((\|).+?)*?)\}")
+
+
+def handle_dynamic_prompt_variants(prompt, repeat_count):
+    founds = list(RE_DYNAMIC_PROMPT.finditer(prompt))
+    if not founds:
+        return [prompt]
+
+    # make each replacement for each variant
+    enumerating = False
+    replacers = []
+    for found in founds:
+        # if "e$$" is found, enumerate all variants
+        found_enumerating = found.group(2) is not None
+        enumerating = enumerating or found_enumerating
+
+        separator = ", " if found.group(6) is None else found.group(6)
+        variants = found.group(7).split("|")
+
+        # parse count range
+        count_range = found.group(4)
+        if count_range is None:
+            count_range = [1, 1]
+        else:
+            count_range = count_range.split("-")
+            if len(count_range) == 1:
+                count_range = [int(count_range[0]), int(count_range[0])]
+            elif len(count_range) == 2:
+                count_range = [int(count_range[0]), int(count_range[1])]
+            else:
+                print(f"invalid count range: {count_range}")
+                count_range = [1, 1]
+            if count_range[0] > count_range[1]:
+                count_range = [count_range[1], count_range[0]]
+            if count_range[0] < 0:
+                count_range[0] = 0
+            if count_range[1] > len(variants):
+                count_range[1] = len(variants)
+
+        if found_enumerating:
+            # make function to enumerate all combinations
+            def make_replacer_enum(vari, cr, sep):
+                def replacer():
+                    values = []
+                    for count in range(cr[0], cr[1] + 1):
+                        for comb in itertools.combinations(vari, count):
+                            values.append(sep.join(comb))
+                    return values
+
+                return replacer
+
+            replacers.append(make_replacer_enum(variants, count_range, separator))
+        else:
+            # make function to choose random combinations
+            def make_replacer_single(vari, cr, sep):
+                def replacer():
+                    count = random.randint(cr[0], cr[1])
+                    comb = random.sample(vari, count)
+                    return [sep.join(comb)]
+
+                return replacer
+
+            replacers.append(make_replacer_single(variants, count_range, separator))
+
+    # make each prompt
+    if not enumerating:
+        # if not enumerating, repeat the prompt, replace each variant randomly
+        prompts = []
+        for _ in range(repeat_count):
+            current = prompt
+            for found, replacer in zip(founds, replacers):
+                current = current.replace(found.group(0), replacer()[0], 1)
+            prompts.append(current)
+    else:
+        # if enumerating, iterate all combinations for previous prompts
+        prompts = [prompt]
+
+        for found, replacer in zip(founds, replacers):
+            if found.group(2) is not None:
+                # make all combinations for existing prompts
+                new_prompts = []
+                for current in prompts:
+                    replecements = replacer()
+                    for replecement in replecements:
+                        new_prompts.append(current.replace(found.group(0), replecement, 1))
+                prompts = new_prompts
+
+        for found, replacer in zip(founds, replacers):
+            # make random selection for existing prompts
+            if found.group(2) is None:
+                for i in range(len(prompts)):
+                    prompts[i] = prompts[i].replace(found.group(0), replacer()[0], 1)
+
+    return prompts
+
+
+# endregion
+
+
+# def load_clip_l14_336(dtype):
+#   print(f"loading CLIP: {CLIP_ID_L14_336}")
+#   text_encoder = CLIPTextModel.from_pretrained(CLIP_ID_L14_336, torch_dtype=dtype)
+#   return text_encoder
+
+
+class BatchDataBase(NamedTuple):
+    # バッチ分割が必要ないデータ
+    step: int
+    prompt: str
+    negative_prompt: str
+    seed: int
+    init_image: Any
+    mask_image: Any
+    clip_prompt: str
+    guide_image: Any
+
+
+class BatchDataExt(NamedTuple):
+    # バッチ分割が必要なデータ
+    width: int
+    height: int
+    steps: int
+    scale: float
+    negative_scale: float
+    strength: float
+    network_muls: Tuple[float]
+    num_sub_prompts: int
+
+
+class BatchData(NamedTuple):
+    return_latents: bool
+    base: BatchDataBase
+    ext: BatchDataExt
+
+
+def main(args):
+    if args.fp16:
+        dtype = torch.float16
+    elif args.bf16:
+        dtype = torch.bfloat16
+    else:
+        dtype = torch.float32
+
+    highres_fix = args.highres_fix_scale is not None
+    # assert not highres_fix or args.image_path is None, f"highres_fix doesn't work with img2img / highres_fixはimg2imgと同時に使えません"
+
+    if args.v_parameterization and not args.v2:
+        print("v_parameterization should be with v2 / v1でv_parameterizationを使用することは想定されていません")
+    if args.v2 and args.clip_skip is not None:
+        print("v2 with clip_skip will be unexpected / v2でclip_skipを使用することは想定されていません")
+
+    # モデルを読み込む
+    if not os.path.isfile(args.ckpt):  # ファイルがないならパターンで探し、一つだけ該当すればそれを使う
+        files = glob.glob(args.ckpt)
+        if len(files) == 1:
+            args.ckpt = files[0]
+
+    use_stable_diffusion_format = os.path.isfile(args.ckpt)
+    if use_stable_diffusion_format:
+        print("load StableDiffusion checkpoint")
+        text_encoder, vae, unet = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.ckpt)
+    else:
+        print("load Diffusers pretrained models")
+        loading_pipe = StableDiffusionPipeline.from_pretrained(args.ckpt, safety_checker=None, torch_dtype=dtype)
+        text_encoder = loading_pipe.text_encoder
+        vae = loading_pipe.vae
+        unet = loading_pipe.unet
+        tokenizer = loading_pipe.tokenizer
+        del loading_pipe
+
+        # Diffusers U-Net to original U-Net
+        original_unet = UNet2DConditionModel(
+            unet.config.sample_size,
+            unet.config.attention_head_dim,
+            unet.config.cross_attention_dim,
+            unet.config.use_linear_projection,
+            unet.config.upcast_attention,
+        )
+        original_unet.load_state_dict(unet.state_dict())
+        unet = original_unet
+
+    # VAEを読み込む
+    if args.vae is not None:
+        vae = model_util.load_vae(args.vae, dtype)
+        print("additional VAE loaded")
+
+    # # 置換するCLIPを読み込む
+    # if args.replace_clip_l14_336:
+    #   text_encoder = load_clip_l14_336(dtype)
+    #   print(f"large clip {CLIP_ID_L14_336} is loaded")
+
+    if args.clip_guidance_scale > 0.0 or args.clip_image_guidance_scale:
+        print("prepare clip model")
+        clip_model = CLIPModel.from_pretrained(CLIP_MODEL_PATH, torch_dtype=dtype)
+    else:
+        clip_model = None
+
+    if args.vgg16_guidance_scale > 0.0:
+        print("prepare resnet model")
+        vgg16_model = torchvision.models.vgg16(torchvision.models.VGG16_Weights.IMAGENET1K_V1)
+    else:
+        vgg16_model = None
+
+    # xformers、Hypernetwork対応
+    if not args.diffusers_xformers:
+        mem_eff = not (args.xformers or args.sdpa)
+        replace_unet_modules(unet, mem_eff, args.xformers, args.sdpa)
+        replace_vae_modules(vae, mem_eff, args.xformers, args.sdpa)
+
+    # tokenizerを読み込む
+    print("loading tokenizer")
+    if use_stable_diffusion_format:
+        tokenizer = train_util.load_tokenizer(args)
+
+    # schedulerを用意する
+    sched_init_args = {}
+    scheduler_num_noises_per_step = 1
+    if args.sampler == "ddim":
+        scheduler_cls = DDIMScheduler
+        scheduler_module = diffusers.schedulers.scheduling_ddim
+    elif args.sampler == "ddpm":  # ddpmはおかしくなるのでoptionから外してある
+        scheduler_cls = DDPMScheduler
+        scheduler_module = diffusers.schedulers.scheduling_ddpm
+    elif args.sampler == "pndm":
+        scheduler_cls = PNDMScheduler
+        scheduler_module = diffusers.schedulers.scheduling_pndm
+    elif args.sampler == "lms" or args.sampler == "k_lms":
+        scheduler_cls = LMSDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_lms_discrete
+    elif args.sampler == "euler" or args.sampler == "k_euler":
+        scheduler_cls = EulerDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_euler_discrete
+    elif args.sampler == "euler_a" or args.sampler == "k_euler_a":
+        scheduler_cls = EulerAncestralDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_euler_ancestral_discrete
+    elif args.sampler == "dpmsolver" or args.sampler == "dpmsolver++":
+        scheduler_cls = DPMSolverMultistepScheduler
+        sched_init_args["algorithm_type"] = args.sampler
+        scheduler_module = diffusers.schedulers.scheduling_dpmsolver_multistep
+    elif args.sampler == "dpmsingle":
+        scheduler_cls = DPMSolverSinglestepScheduler
+        scheduler_module = diffusers.schedulers.scheduling_dpmsolver_singlestep
+    elif args.sampler == "heun":
+        scheduler_cls = HeunDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_heun_discrete
+    elif args.sampler == "dpm_2" or args.sampler == "k_dpm_2":
+        scheduler_cls = KDPM2DiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_k_dpm_2_discrete
+    elif args.sampler == "dpm_2_a" or args.sampler == "k_dpm_2_a":
+        scheduler_cls = KDPM2AncestralDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_k_dpm_2_ancestral_discrete
+        scheduler_num_noises_per_step = 2
+
+    if args.v_parameterization:
+        sched_init_args["prediction_type"] = "v_prediction"
+
+    # samplerの乱数をあらかじめ指定するための処理
+
+    # replace randn
+    class NoiseManager:
+        def __init__(self):
+            self.sampler_noises = None
+            self.sampler_noise_index = 0
+
+        def reset_sampler_noises(self, noises):
+            self.sampler_noise_index = 0
+            self.sampler_noises = noises
+
+        def randn(self, shape, device=None, dtype=None, layout=None, generator=None):
+            # print("replacing", shape, len(self.sampler_noises), self.sampler_noise_index)
+            if self.sampler_noises is not None and self.sampler_noise_index < len(self.sampler_noises):
+                noise = self.sampler_noises[self.sampler_noise_index]
+                if shape != noise.shape:
+                    noise = None
+            else:
+                noise = None
+
+            if noise == None:
+                print(f"unexpected noise request: {self.sampler_noise_index}, {shape}")
+                noise = torch.randn(shape, dtype=dtype, device=device, generator=generator)
+
+            self.sampler_noise_index += 1
+            return noise
+
+    class TorchRandReplacer:
+        def __init__(self, noise_manager):
+            self.noise_manager = noise_manager
+
+        def __getattr__(self, item):
+            if item == "randn":
+                return self.noise_manager.randn
+            if hasattr(torch, item):
+                return getattr(torch, item)
+            raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, item))
+
+    noise_manager = NoiseManager()
+    if scheduler_module is not None:
+        scheduler_module.torch = TorchRandReplacer(noise_manager)
+
+    scheduler = scheduler_cls(
+        num_train_timesteps=SCHEDULER_TIMESTEPS,
+        beta_start=SCHEDULER_LINEAR_START,
+        beta_end=SCHEDULER_LINEAR_END,
+        beta_schedule=SCHEDLER_SCHEDULE,
+        **sched_init_args,
+    )
+
+    # clip_sample=Trueにする
+    if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is False:
+        print("set clip_sample to True")
+        scheduler.config.clip_sample = True
+
+    # deviceを決定する
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # "mps"を考量してない
+
+    # custom pipelineをコピったやつを生成する
+    if args.vae_slices:
+        from library.slicing_vae import SlicingAutoencoderKL
+
+        sli_vae = SlicingAutoencoderKL(
+            act_fn="silu",
+            block_out_channels=(128, 256, 512, 512),
+            down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D"],
+            in_channels=3,
+            latent_channels=4,
+            layers_per_block=2,
+            norm_num_groups=32,
+            out_channels=3,
+            sample_size=512,
+            up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"],
+            num_slices=args.vae_slices,
+        )
+        sli_vae.load_state_dict(vae.state_dict())  # vaeのパラメータをコピーする
+        vae = sli_vae
+        del sli_vae
+    vae.to(dtype).to(device)
+
+    text_encoder.to(dtype).to(device)
+    unet.to(dtype).to(device)
+    if clip_model is not None:
+        clip_model.to(dtype).to(device)
+    if vgg16_model is not None:
+        vgg16_model.to(dtype).to(device)
+
+    # networkを組み込む
+    if args.network_module:
+        networks = []
+        network_default_muls = []
+        network_pre_calc = args.network_pre_calc
+
+        for i, network_module in enumerate(args.network_module):
+            print("import network module:", network_module)
+            imported_module = importlib.import_module(network_module)
+
+            network_mul = 1.0 if args.network_mul is None or len(args.network_mul) <= i else args.network_mul[i]
+            network_default_muls.append(network_mul)
+
+            net_kwargs = {}
+            if args.network_args and i < len(args.network_args):
+                network_args = args.network_args[i]
+                # TODO escape special chars
+                network_args = network_args.split(";")
+                for net_arg in network_args:
+                    key, value = net_arg.split("=")
+                    net_kwargs[key] = value
+
+            if args.network_weights and i < len(args.network_weights):
+                network_weight = args.network_weights[i]
+                print("load network weights from:", network_weight)
+
+                if model_util.is_safetensors(network_weight) and args.network_show_meta:
+                    from safetensors.torch import safe_open
+
+                    with safe_open(network_weight, framework="pt") as f:
+                        metadata = f.metadata()
+                    if metadata is not None:
+                        print(f"metadata for: {network_weight}: {metadata}")
+
+                network, weights_sd = imported_module.create_network_from_weights(
+                    network_mul, network_weight, vae, text_encoder, unet, for_inference=True, **net_kwargs
+                )
+            else:
+                raise ValueError("No weight. Weight is required.")
+            if network is None:
+                return
+
+            mergeable = network.is_mergeable()
+            if args.network_merge and not mergeable:
+                print("network is not mergiable. ignore merge option.")
+
+            if not args.network_merge or not mergeable:
+                network.apply_to(text_encoder, unet)
+                info = network.load_state_dict(weights_sd, False)  # network.load_weightsを使うようにするとよい
+                print(f"weights are loaded: {info}")
+
+                if args.opt_channels_last:
+                    network.to(memory_format=torch.channels_last)
+                network.to(dtype).to(device)
+
+                if network_pre_calc:
+                    print("backup original weights")
+                    network.backup_weights()
+
+                networks.append(network)
+            else:
+                network.merge_to(text_encoder, unet, weights_sd, dtype, device)
+
+    else:
+        networks = []
+
+    # upscalerの指定があれば取得する
+    upscaler = None
+    if args.highres_fix_upscaler:
+        print("import upscaler module:", args.highres_fix_upscaler)
+        imported_module = importlib.import_module(args.highres_fix_upscaler)
+
+        us_kwargs = {}
+        if args.highres_fix_upscaler_args:
+            for net_arg in args.highres_fix_upscaler_args.split(";"):
+                key, value = net_arg.split("=")
+                us_kwargs[key] = value
+
+        print("create upscaler")
+        upscaler = imported_module.create_upscaler(**us_kwargs)
+        upscaler.to(dtype).to(device)
+
+    # ControlNetの処理
+    control_nets: List[ControlNetInfo] = []
+    if args.control_net_models:
+        for i, model in enumerate(args.control_net_models):
+            prep_type = None if not args.control_net_preps or len(args.control_net_preps) <= i else args.control_net_preps[i]
+            weight = 1.0 if not args.control_net_weights or len(args.control_net_weights) <= i else args.control_net_weights[i]
+            ratio = 1.0 if not args.control_net_ratios or len(args.control_net_ratios) <= i else args.control_net_ratios[i]
+
+            ctrl_unet, ctrl_net = original_control_net.load_control_net(args.v2, unet, model)
+            prep = original_control_net.load_preprocess(prep_type)
+            control_nets.append(ControlNetInfo(ctrl_unet, ctrl_net, prep, weight, ratio))
+
+    if args.opt_channels_last:
+        print(f"set optimizing: channels last")
+        text_encoder.to(memory_format=torch.channels_last)
+        vae.to(memory_format=torch.channels_last)
+        unet.to(memory_format=torch.channels_last)
+        if clip_model is not None:
+            clip_model.to(memory_format=torch.channels_last)
+        if networks:
+            for network in networks:
+                network.to(memory_format=torch.channels_last)
+        if vgg16_model is not None:
+            vgg16_model.to(memory_format=torch.channels_last)
+
+        for cn in control_nets:
+            cn.unet.to(memory_format=torch.channels_last)
+            cn.net.to(memory_format=torch.channels_last)
+
+    pipe = PipelineLike(
+        device,
+        vae,
+        text_encoder,
+        tokenizer,
+        unet,
+        scheduler,
+        args.clip_skip,
+        clip_model,
+        args.clip_guidance_scale,
+        args.clip_image_guidance_scale,
+        vgg16_model,
+        args.vgg16_guidance_scale,
+        args.vgg16_guidance_layer,
+    )
+    pipe.set_control_nets(control_nets)
+    print("pipeline is ready.")
+
+    if args.diffusers_xformers:
+        pipe.enable_xformers_memory_efficient_attention()
+
+    # Extended Textual Inversion および Textual Inversionを処理する
+    if args.XTI_embeddings:
+        diffusers.models.UNet2DConditionModel.forward = unet_forward_XTI
+        diffusers.models.unet_2d_blocks.CrossAttnDownBlock2D.forward = downblock_forward_XTI
+        diffusers.models.unet_2d_blocks.CrossAttnUpBlock2D.forward = upblock_forward_XTI
+
+    if args.textual_inversion_embeddings:
+        token_ids_embeds = []
+        for embeds_file in args.textual_inversion_embeddings:
+            if model_util.is_safetensors(embeds_file):
+                from safetensors.torch import load_file
+
+                data = load_file(embeds_file)
+            else:
+                data = torch.load(embeds_file, map_location="cpu")
+
+            if "string_to_param" in data:
+                data = data["string_to_param"]
+            embeds = next(iter(data.values()))
+
+            if type(embeds) != torch.Tensor:
+                raise ValueError(f"weight file does not contains Tensor / 重みファイルのデータがTensorではありません: {embeds_file}")
+
+            num_vectors_per_token = embeds.size()[0]
+            token_string = os.path.splitext(os.path.basename(embeds_file))[0]
+            token_strings = [token_string] + [f"{token_string}{i+1}" for i in range(num_vectors_per_token - 1)]
+
+            # add new word to tokenizer, count is num_vectors_per_token
+            num_added_tokens = tokenizer.add_tokens(token_strings)
+            assert (
+                num_added_tokens == num_vectors_per_token
+            ), f"tokenizer has same word to token string (filename). please rename the file / 指定した名前（ファイル名）のトークンが既に存在します。ファイルをリネームしてください: {embeds_file}"
+
+            token_ids = tokenizer.convert_tokens_to_ids(token_strings)
+            print(f"Textual Inversion embeddings `{token_string}` loaded. Tokens are added: {token_ids}")
+            assert (
+                min(token_ids) == token_ids[0] and token_ids[-1] == token_ids[0] + len(token_ids) - 1
+            ), f"token ids is not ordered"
+            assert len(tokenizer) - 1 == token_ids[-1], f"token ids is not end of tokenize: {len(tokenizer)}"
+
+            if num_vectors_per_token > 1:
+                pipe.add_token_replacement(token_ids[0], token_ids)
+
+            token_ids_embeds.append((token_ids, embeds))
+
+        text_encoder.resize_token_embeddings(len(tokenizer))
+        token_embeds = text_encoder.get_input_embeddings().weight.data
+        for token_ids, embeds in token_ids_embeds:
+            for token_id, embed in zip(token_ids, embeds):
+                token_embeds[token_id] = embed
+
+    if args.XTI_embeddings:
+        XTI_layers = [
+            "IN01",
+            "IN02",
+            "IN04",
+            "IN05",
+            "IN07",
+            "IN08",
+            "MID",
+            "OUT03",
+            "OUT04",
+            "OUT05",
+            "OUT06",
+            "OUT07",
+            "OUT08",
+            "OUT09",
+            "OUT10",
+            "OUT11",
+        ]
+        token_ids_embeds_XTI = []
+        for embeds_file in args.XTI_embeddings:
+            if model_util.is_safetensors(embeds_file):
+                from safetensors.torch import load_file
+
+                data = load_file(embeds_file)
+            else:
+                data = torch.load(embeds_file, map_location="cpu")
+            if set(data.keys()) != set(XTI_layers):
+                raise ValueError("NOT XTI")
+            embeds = torch.concat(list(data.values()))
+            num_vectors_per_token = data["MID"].size()[0]
+
+            token_string = os.path.splitext(os.path.basename(embeds_file))[0]
+            token_strings = [token_string] + [f"{token_string}{i+1}" for i in range(num_vectors_per_token - 1)]
+
+            # add new word to tokenizer, count is num_vectors_per_token
+            num_added_tokens = tokenizer.add_tokens(token_strings)
+            assert (
+                num_added_tokens == num_vectors_per_token
+            ), f"tokenizer has same word to token string (filename). please rename the file / 指定した名前（ファイル名）のトークンが既に存在します。ファイルをリネームしてください: {embeds_file}"
+
+            token_ids = tokenizer.convert_tokens_to_ids(token_strings)
+            print(f"XTI embeddings `{token_string}` loaded. Tokens are added: {token_ids}")
+
+            # if num_vectors_per_token > 1:
+            pipe.add_token_replacement(token_ids[0], token_ids)
+
+            token_strings_XTI = []
+            for layer_name in XTI_layers:
+                token_strings_XTI += [f"{t}_{layer_name}" for t in token_strings]
+            tokenizer.add_tokens(token_strings_XTI)
+            token_ids_XTI = tokenizer.convert_tokens_to_ids(token_strings_XTI)
+            token_ids_embeds_XTI.append((token_ids_XTI, embeds))
+            for t in token_ids:
+                t_XTI_dic = {}
+                for i, layer_name in enumerate(XTI_layers):
+                    t_XTI_dic[layer_name] = t + (i + 1) * num_added_tokens
+                pipe.add_token_replacement_XTI(t, t_XTI_dic)
+
+            text_encoder.resize_token_embeddings(len(tokenizer))
+            token_embeds = text_encoder.get_input_embeddings().weight.data
+            for token_ids, embeds in token_ids_embeds_XTI:
+                for token_id, embed in zip(token_ids, embeds):
+                    token_embeds[token_id] = embed
+
+    # promptを取得する
+    if args.from_file is not None:
+        print(f"reading prompts from {args.from_file}")
+        with open(args.from_file, "r", encoding="utf-8") as f:
+            prompt_list = f.read().splitlines()
+            prompt_list = [d for d in prompt_list if len(d.strip()) > 0]
+    elif args.prompt is not None:
+        prompt_list = [args.prompt]
+    else:
+        prompt_list = []
+
+    if args.interactive:
+        args.n_iter = 1
+
+    # img2imgの前処理、画像の読み込みなど
+    def load_images(path):
+        if os.path.isfile(path):
+            paths = [path]
+        else:
+            paths = (
+                glob.glob(os.path.join(path, "*.png"))
+                + glob.glob(os.path.join(path, "*.jpg"))
+                + glob.glob(os.path.join(path, "*.jpeg"))
+                + glob.glob(os.path.join(path, "*.webp"))
+            )
+            paths.sort()
+
+        images = []
+        for p in paths:
+            image = Image.open(p)
+            if image.mode != "RGB":
+                print(f"convert image to RGB from {image.mode}: {p}")
+                image = image.convert("RGB")
+            images.append(image)
+
+        return images
+
+    def resize_images(imgs, size):
+        resized = []
+        for img in imgs:
+            r_img = img.resize(size, Image.Resampling.LANCZOS)
+            if hasattr(img, "filename"):  # filename属性がない場合があるらしい
+                r_img.filename = img.filename
+            resized.append(r_img)
+        return resized
+
+    if args.image_path is not None:
+        print(f"load image for img2img: {args.image_path}")
+        init_images = load_images(args.image_path)
+        assert len(init_images) > 0, f"No image / 画像がありません: {args.image_path}"
+        print(f"loaded {len(init_images)} images for img2img")
+    else:
+        init_images = None
+
+    if args.mask_path is not None:
+        print(f"load mask for inpainting: {args.mask_path}")
+        mask_images = load_images(args.mask_path)
+        assert len(mask_images) > 0, f"No mask image / マスク画像がありません: {args.image_path}"
+        print(f"loaded {len(mask_images)} mask images for inpainting")
+    else:
+        mask_images = None
+
+    # promptがないとき、画像のPngInfoから取得する
+    if init_images is not None and len(prompt_list) == 0 and not args.interactive:
+        print("get prompts from images' meta data")
+        for img in init_images:
+            if "prompt" in img.text:
+                prompt = img.text["prompt"]
+                if "negative-prompt" in img.text:
+                    prompt += " --n " + img.text["negative-prompt"]
+                prompt_list.append(prompt)
+
+        # プロンプトと画像を一致させるため指定回数だけ繰り返す（画像を増幅する）
+        l = []
+        for im in init_images:
+            l.extend([im] * args.images_per_prompt)
+        init_images = l
+
+        if mask_images is not None:
+            l = []
+            for im in mask_images:
+                l.extend([im] * args.images_per_prompt)
+            mask_images = l
+
+    # 画像サイズにオプション指定があるときはリサイズする
+    if args.W is not None and args.H is not None:
+        # highres fix を考慮に入れる
+        w, h = args.W, args.H
+        if highres_fix:
+            w = int(w * args.highres_fix_scale + 0.5)
+            h = int(h * args.highres_fix_scale + 0.5)
+
+        if init_images is not None:
+            print(f"resize img2img source images to {w}*{h}")
+            init_images = resize_images(init_images, (w, h))
+        if mask_images is not None:
+            print(f"resize img2img mask images to {w}*{h}")
+            mask_images = resize_images(mask_images, (w, h))
+
+    regional_network = False
+    if networks and mask_images:
+        # mask を領域情報として流用する、現在は一回のコマンド呼び出しで1枚だけ対応
+        regional_network = True
+        print("use mask as region")
+
+        size = None
+        for i, network in enumerate(networks):
+            if i < 3:
+                np_mask = np.array(mask_images[0])
+                np_mask = np_mask[:, :, i]
+                size = np_mask.shape
+            else:
+                np_mask = np.full(size, 255, dtype=np.uint8)
+            mask = torch.from_numpy(np_mask.astype(np.float32) / 255.0)
+            network.set_region(i, i == len(networks) - 1, mask)
+        mask_images = None
+
+    prev_image = None  # for VGG16 guided
+    if args.guide_image_path is not None:
+        print(f"load image for CLIP/VGG16/ControlNet guidance: {args.guide_image_path}")
+        guide_images = []
+        for p in args.guide_image_path:
+            guide_images.extend(load_images(p))
+
+        print(f"loaded {len(guide_images)} guide images for guidance")
+        if len(guide_images) == 0:
+            print(f"No guide image, use previous generated image. / ガイド画像がありません。直前に生成した画像を使います: {args.image_path}")
+            guide_images = None
+    else:
+        guide_images = None
+
+    # seed指定時はseedを決めておく
+    if args.seed is not None:
+        # dynamic promptを使うと足りなくなる→images_per_promptを適当に大きくしておいてもらう
+        random.seed(args.seed)
+        predefined_seeds = [random.randint(0, 0x7FFFFFFF) for _ in range(args.n_iter * len(prompt_list) * args.images_per_prompt)]
+        if len(predefined_seeds) == 1:
+            predefined_seeds[0] = args.seed
+    else:
+        predefined_seeds = None
+
+    # デフォルト画像サイズを設定する：img2imgではこれらの値は無視される（またはW*Hにリサイズ済み）
+    if args.W is None:
+        args.W = 512
+    if args.H is None:
+        args.H = 512
+
+    # 画像生成のループ
+    os.makedirs(args.outdir, exist_ok=True)
+    max_embeddings_multiples = 1 if args.max_embeddings_multiples is None else args.max_embeddings_multiples
+
+    for gen_iter in range(args.n_iter):
+        print(f"iteration {gen_iter+1}/{args.n_iter}")
+        iter_seed = random.randint(0, 0x7FFFFFFF)
+
+        # バッチ処理の関数
+        def process_batch(batch: List[BatchData], highres_fix, highres_1st=False):
+            batch_size = len(batch)
+
+            # highres_fixの処理
+            if highres_fix and not highres_1st:
+                # 1st stageのバッチを作成して呼び出す：サイズを小さくして呼び出す
+                is_1st_latent = upscaler.support_latents() if upscaler else args.highres_fix_latents_upscaling
+
+                print("process 1st stage")
+                batch_1st = []
+                for _, base, ext in batch:
+                    width_1st = int(ext.width * args.highres_fix_scale + 0.5)
+                    height_1st = int(ext.height * args.highres_fix_scale + 0.5)
+                    width_1st = width_1st - width_1st % 32
+                    height_1st = height_1st - height_1st % 32
+
+                    strength_1st = ext.strength if args.highres_fix_strength is None else args.highres_fix_strength
+
+                    ext_1st = BatchDataExt(
+                        width_1st,
+                        height_1st,
+                        args.highres_fix_steps,
+                        ext.scale,
+                        ext.negative_scale,
+                        strength_1st,
+                        ext.network_muls,
+                        ext.num_sub_prompts,
+                    )
+                    batch_1st.append(BatchData(is_1st_latent, base, ext_1st))
+
+                pipe.set_enable_control_net(True)  # 1st stageではControlNetを有効にする
+                images_1st = process_batch(batch_1st, True, True)
+
+                # 2nd stageのバッチを作成して以下処理する
+                print("process 2nd stage")
+                width_2nd, height_2nd = batch[0].ext.width, batch[0].ext.height
+
+                if upscaler:
+                    # upscalerを使って画像を拡大する
+                    lowreso_imgs = None if is_1st_latent else images_1st
+                    lowreso_latents = None if not is_1st_latent else images_1st
+
+                    # 戻り値はPIL.Image.Imageかtorch.Tensorのlatents
+                    batch_size = len(images_1st)
+                    vae_batch_size = (
+                        batch_size
+                        if args.vae_batch_size is None
+                        else (max(1, int(batch_size * args.vae_batch_size)) if args.vae_batch_size < 1 else args.vae_batch_size)
+                    )
+                    vae_batch_size = int(vae_batch_size)
+                    images_1st = upscaler.upscale(
+                        vae, lowreso_imgs, lowreso_latents, dtype, width_2nd, height_2nd, batch_size, vae_batch_size
+                    )
+
+                elif args.highres_fix_latents_upscaling:
+                    # latentを拡大する
+                    org_dtype = images_1st.dtype
+                    if images_1st.dtype == torch.bfloat16:
+                        images_1st = images_1st.to(torch.float)  # interpolateがbf16をサポートしていない
+                    images_1st = torch.nn.functional.interpolate(
+                        images_1st, (batch[0].ext.height // 8, batch[0].ext.width // 8), mode="bilinear"
+                    )  # , antialias=True)
+                    images_1st = images_1st.to(org_dtype)
+
+                else:
+                    # 画像をLANCZOSで拡大する
+                    images_1st = [image.resize((width_2nd, height_2nd), resample=PIL.Image.LANCZOS) for image in images_1st]
+
+                batch_2nd = []
+                for i, (bd, image) in enumerate(zip(batch, images_1st)):
+                    bd_2nd = BatchData(False, BatchDataBase(*bd.base[0:3], bd.base.seed + 1, image, None, *bd.base[6:]), bd.ext)
+                    batch_2nd.append(bd_2nd)
+                batch = batch_2nd
+
+                if args.highres_fix_disable_control_net:
+                    pipe.set_enable_control_net(False)  # オプション指定時、2nd stageではControlNetを無効にする
+
+            # このバッチの情報を取り出す
+            (
+                return_latents,
+                (step_first, _, _, _, init_image, mask_image, _, guide_image),
+                (width, height, steps, scale, negative_scale, strength, network_muls, num_sub_prompts),
+            ) = batch[0]
+            noise_shape = (LATENT_CHANNELS, height // DOWNSAMPLING_FACTOR, width // DOWNSAMPLING_FACTOR)
+
+            prompts = []
+            negative_prompts = []
+            start_code = torch.zeros((batch_size, *noise_shape), device=device, dtype=dtype)
+            noises = [
+                torch.zeros((batch_size, *noise_shape), device=device, dtype=dtype)
+                for _ in range(steps * scheduler_num_noises_per_step)
+            ]
+            seeds = []
+            clip_prompts = []
+
+            if init_image is not None:  # img2img?
+                i2i_noises = torch.zeros((batch_size, *noise_shape), device=device, dtype=dtype)
+                init_images = []
+
+                if mask_image is not None:
+                    mask_images = []
+                else:
+                    mask_images = None
+            else:
+                i2i_noises = None
+                init_images = None
+                mask_images = None
+
+            if guide_image is not None:  # CLIP image guided?
+                guide_images = []
+            else:
+                guide_images = None
+
+            # バッチ内の位置に関わらず同じ乱数を使うためにここで乱数を生成しておく。あわせてimage/maskがbatch内で同一かチェックする
+            all_images_are_same = True
+            all_masks_are_same = True
+            all_guide_images_are_same = True
+            for i, (_, (_, prompt, negative_prompt, seed, init_image, mask_image, clip_prompt, guide_image), _) in enumerate(batch):
+                prompts.append(prompt)
+                negative_prompts.append(negative_prompt)
+                seeds.append(seed)
+                clip_prompts.append(clip_prompt)
+
+                if init_image is not None:
+                    init_images.append(init_image)
+                    if i > 0 and all_images_are_same:
+                        all_images_are_same = init_images[-2] is init_image
+
+                if mask_image is not None:
+                    mask_images.append(mask_image)
+                    if i > 0 and all_masks_are_same:
+                        all_masks_are_same = mask_images[-2] is mask_image
+
+                if guide_image is not None:
+                    if type(guide_image) is list:
+                        guide_images.extend(guide_image)
+                        all_guide_images_are_same = False
+                    else:
+                        guide_images.append(guide_image)
+                        if i > 0 and all_guide_images_are_same:
+                            all_guide_images_are_same = guide_images[-2] is guide_image
+
+                # make start code
+                torch.manual_seed(seed)
+                start_code[i] = torch.randn(noise_shape, device=device, dtype=dtype)
+
+                # make each noises
+                for j in range(steps * scheduler_num_noises_per_step):
+                    noises[j][i] = torch.randn(noise_shape, device=device, dtype=dtype)
+
+                if i2i_noises is not None:  # img2img noise
+                    i2i_noises[i] = torch.randn(noise_shape, device=device, dtype=dtype)
+
+            noise_manager.reset_sampler_noises(noises)
+
+            # すべての画像が同じなら1枚だけpipeに渡すことでpipe側で処理を高速化する
+            if init_images is not None and all_images_are_same:
+                init_images = init_images[0]
+            if mask_images is not None and all_masks_are_same:
+                mask_images = mask_images[0]
+            if guide_images is not None and all_guide_images_are_same:
+                guide_images = guide_images[0]
+
+            # ControlNet使用時はguide imageをリサイズする
+            if control_nets:
+                # TODO resampleのメソッド
+                guide_images = guide_images if type(guide_images) == list else [guide_images]
+                guide_images = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in guide_images]
+                if len(guide_images) == 1:
+                    guide_images = guide_images[0]
+
+            # generate
+            if networks:
+                # 追加ネットワークの処理
+                shared = {}
+                for n, m in zip(networks, network_muls if network_muls else network_default_muls):
+                    n.set_multiplier(m)
+                    if regional_network:
+                        n.set_current_generation(batch_size, num_sub_prompts, width, height, shared)
+
+                if not regional_network and network_pre_calc:
+                    for n in networks:
+                        n.restore_weights()
+                    for n in networks:
+                        n.pre_calculation()
+                    print("pre-calculation... done")
+
+            images = pipe(
+                prompts,
+                negative_prompts,
+                init_images,
+                mask_images,
+                height,
+                width,
+                steps,
+                scale,
+                negative_scale,
+                strength,
+                latents=start_code,
+                output_type="pil",
+                max_embeddings_multiples=max_embeddings_multiples,
+                img2img_noise=i2i_noises,
+                vae_batch_size=args.vae_batch_size,
+                return_latents=return_latents,
+                clip_prompts=clip_prompts,
+                clip_guide_images=guide_images,
+            )[0]
+            if highres_1st and not args.highres_fix_save_1st:  # return images or latents
+                return images
+
+            # save image
+            highres_prefix = ("0" if highres_1st else "1") if highres_fix else ""
+            ts_str = time.strftime("%Y%m%d%H%M%S", time.localtime())
+            for i, (image, prompt, negative_prompts, seed, clip_prompt) in enumerate(
+                zip(images, prompts, negative_prompts, seeds, clip_prompts)
+            ):
+                metadata = PngInfo()
+                metadata.add_text("prompt", prompt)
+                metadata.add_text("seed", str(seed))
+                metadata.add_text("sampler", args.sampler)
+                metadata.add_text("steps", str(steps))
+                metadata.add_text("scale", str(scale))
+                if negative_prompt is not None:
+                    metadata.add_text("negative-prompt", negative_prompt)
+                if negative_scale is not None:
+                    metadata.add_text("negative-scale", str(negative_scale))
+                if clip_prompt is not None:
+                    metadata.add_text("clip-prompt", clip_prompt)
+
+                if args.use_original_file_name and init_images is not None:
+                    if type(init_images) is list:
+                        fln = os.path.splitext(os.path.basename(init_images[i % len(init_images)].filename))[0] + ".png"
+                    else:
+                        fln = os.path.splitext(os.path.basename(init_images.filename))[0] + ".png"
+                elif args.sequential_file_name:
+                    fln = f"im_{highres_prefix}{step_first + i + 1:06d}.png"
+                else:
+                    fln = f"im_{ts_str}_{highres_prefix}{i:03d}_{seed}.png"
+
+                image.save(os.path.join(args.outdir, fln), pnginfo=metadata)
+
+            if not args.no_preview and not highres_1st and args.interactive:
+                try:
+                    import cv2
+
+                    for prompt, image in zip(prompts, images):
+                        cv2.imshow(prompt[:128], np.array(image)[:, :, ::-1])  # プロンプトが長いと死ぬ
+                        cv2.waitKey()
+                        cv2.destroyAllWindows()
+                except ImportError:
+                    print("opencv-python is not installed, cannot preview / opencv-pythonがインストールされていないためプレビューできません")
+
+            return images
+
+        # 画像生成のプロンプトが一周するまでのループ
+        prompt_index = 0
+        global_step = 0
+        batch_data = []
+        while args.interactive or prompt_index < len(prompt_list):
+            if len(prompt_list) == 0:
+                # interactive
+                valid = False
+                while not valid:
+                    print("\nType prompt:")
+                    try:
+                        raw_prompt = input()
+                    except EOFError:
+                        break
+
+                    valid = len(raw_prompt.strip().split(" --")[0].strip()) > 0
+                if not valid:  # EOF, end app
+                    break
+            else:
+                raw_prompt = prompt_list[prompt_index]
+
+            # sd-dynamic-prompts like variants:
+            # count is 1 (not dynamic) or images_per_prompt (no enumeration) or arbitrary (enumeration)
+            raw_prompts = handle_dynamic_prompt_variants(raw_prompt, args.images_per_prompt)
+
+            # repeat prompt
+            for pi in range(args.images_per_prompt if len(raw_prompts) == 1 else len(raw_prompts)):
+                raw_prompt = raw_prompts[pi] if len(raw_prompts) > 1 else raw_prompts[0]
+
+                if pi == 0 or len(raw_prompts) > 1:
+                    # parse prompt: if prompt is not changed, skip parsing
+                    width = args.W
+                    height = args.H
+                    scale = args.scale
+                    negative_scale = args.negative_scale
+                    steps = args.steps
+                    seed = None
+                    seeds = None
+                    strength = 0.8 if args.strength is None else args.strength
+                    negative_prompt = ""
+                    clip_prompt = None
+                    network_muls = None
+
+                    prompt_args = raw_prompt.strip().split(" --")
+                    prompt = prompt_args[0]
+                    print(f"prompt {prompt_index+1}/{len(prompt_list)}: {prompt}")
+
+                    for parg in prompt_args[1:]:
+                        try:
+                            m = re.match(r"w (\d+)", parg, re.IGNORECASE)
+                            if m:
+                                width = int(m.group(1))
+                                print(f"width: {width}")
+                                continue
+
+                            m = re.match(r"h (\d+)", parg, re.IGNORECASE)
+                            if m:
+                                height = int(m.group(1))
+                                print(f"height: {height}")
+                                continue
+
+                            m = re.match(r"s (\d+)", parg, re.IGNORECASE)
+                            if m:  # steps
+                                steps = max(1, min(1000, int(m.group(1))))
+                                print(f"steps: {steps}")
+                                continue
+
+                            m = re.match(r"d ([\d,]+)", parg, re.IGNORECASE)
+                            if m:  # seed
+                                seeds = [int(d) for d in m.group(1).split(",")]
+                                print(f"seeds: {seeds}")
+                                continue
+
+                            m = re.match(r"l ([\d\.]+)", parg, re.IGNORECASE)
+                            if m:  # scale
+                                scale = float(m.group(1))
+                                print(f"scale: {scale}")
+                                continue
+
+                            m = re.match(r"nl ([\d\.]+|none|None)", parg, re.IGNORECASE)
+                            if m:  # negative scale
+                                if m.group(1).lower() == "none":
+                                    negative_scale = None
+                                else:
+                                    negative_scale = float(m.group(1))
+                                print(f"negative scale: {negative_scale}")
+                                continue
+
+                            m = re.match(r"t ([\d\.]+)", parg, re.IGNORECASE)
+                            if m:  # strength
+                                strength = float(m.group(1))
+                                print(f"strength: {strength}")
+                                continue
+
+                            m = re.match(r"n (.+)", parg, re.IGNORECASE)
+                            if m:  # negative prompt
+                                negative_prompt = m.group(1)
+                                print(f"negative prompt: {negative_prompt}")
+                                continue
+
+                            m = re.match(r"c (.+)", parg, re.IGNORECASE)
+                            if m:  # clip prompt
+                                clip_prompt = m.group(1)
+                                print(f"clip prompt: {clip_prompt}")
+                                continue
+
+                            m = re.match(r"am ([\d\.\-,]+)", parg, re.IGNORECASE)
+                            if m:  # network multiplies
+                                network_muls = [float(v) for v in m.group(1).split(",")]
+                                while len(network_muls) < len(networks):
+                                    network_muls.append(network_muls[-1])
+                                print(f"network mul: {network_muls}")
+                                continue
+
+                        except ValueError as ex:
+                            print(f"Exception in parsing / 解析エラー: {parg}")
+                            print(ex)
+
+                # prepare seed
+                if seeds is not None:  # given in prompt
+                    # 数が足りないなら前のをそのまま使う
+                    if len(seeds) > 0:
+                        seed = seeds.pop(0)
+                else:
+                    if predefined_seeds is not None:
+                        if len(predefined_seeds) > 0:
+                            seed = predefined_seeds.pop(0)
+                        else:
+                            print("predefined seeds are exhausted")
+                            seed = None
+                    elif args.iter_same_seed:
+                        seeds = iter_seed
+                    else:
+                        seed = None  # 前のを消す
+
+                if seed is None:
+                    seed = random.randint(0, 0x7FFFFFFF)
+                if args.interactive:
+                    print(f"seed: {seed}")
+
+                # prepare init image, guide image and mask
+                init_image = mask_image = guide_image = None
+
+                # 同一イメージを使うとき、本当はlatentに変換しておくと無駄がないが面倒なのでとりあえず毎回処理する
+                if init_images is not None:
+                    init_image = init_images[global_step % len(init_images)]
+
+                    # img2imgの場合は、基本的に元画像のサイズで生成する。highres fixの場合はargs.W, args.Hとscaleに従いリサイズ済みなので無視する
+                    # 32単位に丸めたやつにresizeされるので踏襲する
+                    if not highres_fix:
+                        width, height = init_image.size
+                        width = width - width % 32
+                        height = height - height % 32
+                        if width != init_image.size[0] or height != init_image.size[1]:
+                            print(
+                                f"img2img image size is not divisible by 32 so aspect ratio is changed / img2imgの画像サイズが32で割り切れないためリサイズされます。画像が歪みます"
+                            )
+
+                if mask_images is not None:
+                    mask_image = mask_images[global_step % len(mask_images)]
+
+                if guide_images is not None:
+                    if control_nets:  # 複数件の場合あり
+                        c = len(control_nets)
+                        p = global_step % (len(guide_images) // c)
+                        guide_image = guide_images[p * c : p * c + c]
+                    else:
+                        guide_image = guide_images[global_step % len(guide_images)]
+                elif args.clip_image_guidance_scale > 0 or args.vgg16_guidance_scale > 0:
+                    if prev_image is None:
+                        print("Generate 1st image without guide image.")
+                    else:
+                        print("Use previous image as guide image.")
+                        guide_image = prev_image
+
+                if regional_network:
+                    num_sub_prompts = len(prompt.split(" AND "))
+                    assert (
+                        len(networks) <= num_sub_prompts
+                    ), "Number of networks must be less than or equal to number of sub prompts."
+                else:
+                    num_sub_prompts = None
+
+                b1 = BatchData(
+                    False,
+                    BatchDataBase(global_step, prompt, negative_prompt, seed, init_image, mask_image, clip_prompt, guide_image),
+                    BatchDataExt(
+                        width,
+                        height,
+                        steps,
+                        scale,
+                        negative_scale,
+                        strength,
+                        tuple(network_muls) if network_muls else None,
+                        num_sub_prompts,
+                    ),
+                )
+                if len(batch_data) > 0 and batch_data[-1].ext != b1.ext:  # バッチ分割必要？
+                    process_batch(batch_data, highres_fix)
+                    batch_data.clear()
+
+                batch_data.append(b1)
+                if len(batch_data) == args.batch_size:
+                    prev_image = process_batch(batch_data, highres_fix)[0]
+                    batch_data.clear()
+
+                global_step += 1
+
+            prompt_index += 1
+
+        if len(batch_data) > 0:
+            process_batch(batch_data, highres_fix)
+            batch_data.clear()
+
+    print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--v2", action="store_true", help="load Stable Diffusion v2.0 model / Stable Diffusion 2.0のモデルを読み込む")
+    parser.add_argument(
+        "--v_parameterization", action="store_true", help="enable v-parameterization training / v-parameterization学習を有効にする"
+    )
+    parser.add_argument("--prompt", type=str, default=None, help="prompt / プロンプト")
+    parser.add_argument(
+        "--from_file", type=str, default=None, help="if specified, load prompts from this file / 指定時はプロンプトをファイルから読み込む"
+    )
+    parser.add_argument(
+        "--interactive", action="store_true", help="interactive mode (generates one image) / 対話モード（生成される画像は1枚になります）"
+    )
+    parser.add_argument(
+        "--no_preview", action="store_true", help="do not show generated image in interactive mode / 対話モードで画像を表示しない"
+    )
+    parser.add_argument(
+        "--image_path", type=str, default=None, help="image to inpaint or to generate from / img2imgまたはinpaintを行う元画像"
+    )
+    parser.add_argument("--mask_path", type=str, default=None, help="mask in inpainting / inpaint時のマスク")
+    parser.add_argument("--strength", type=float, default=None, help="img2img strength / img2img時のstrength")
+    parser.add_argument("--images_per_prompt", type=int, default=1, help="number of images per prompt / プロンプトあたりの出力枚数")
+    parser.add_argument("--outdir", type=str, default="outputs", help="dir to write results to / 生成画像の出力先")
+    parser.add_argument("--sequential_file_name", action="store_true", help="sequential output file name / 生成画像のファイル名を連番にする")
+    parser.add_argument(
+        "--use_original_file_name",
+        action="store_true",
+        help="prepend original file name in img2img / img2imgで元画像のファイル名を生成画像のファイル名の先頭に付ける",
+    )
+    # parser.add_argument("--ddim_eta", type=float, default=0.0, help="ddim eta (eta=0.0 corresponds to deterministic sampling", )
+    parser.add_argument("--n_iter", type=int, default=1, help="sample this often / 繰り返し回数")
+    parser.add_argument("--H", type=int, default=None, help="image height, in pixel space / 生成画像高さ")
+    parser.add_argument("--W", type=int, default=None, help="image width, in pixel space / 生成画像幅")
+    parser.add_argument("--batch_size", type=int, default=1, help="batch size / バッチサイズ")
+    parser.add_argument(
+        "--vae_batch_size",
+        type=float,
+        default=None,
+        help="batch size for VAE, < 1.0 for ratio / VAE処理時のバッチサイズ、1未満の値の場合は通常バッチサイズの比率",
+    )
+    parser.add_argument(
+        "--vae_slices",
+        type=int,
+        default=None,
+        help="number of slices to split image into for VAE to reduce VRAM usage, None for no splitting (default), slower if specified. 16 or 32 recommended / VAE処理時にVRAM使用量削減のため画像を分割するスライス数、Noneの場合は分割しない（デフォルト）、指定すると遅くなる。16か32程度を推奨",
+    )
+    parser.add_argument("--steps", type=int, default=50, help="number of ddim sampling steps / サンプリングステップ数")
+    parser.add_argument(
+        "--sampler",
+        type=str,
+        default="ddim",
+        choices=[
+            "ddim",
+            "pndm",
+            "lms",
+            "euler",
+            "euler_a",
+            "heun",
+            "dpm_2",
+            "dpm_2_a",
+            "dpmsolver",
+            "dpmsolver++",
+            "dpmsingle",
+            "k_lms",
+            "k_euler",
+            "k_euler_a",
+            "k_dpm_2",
+            "k_dpm_2_a",
+        ],
+        help=f"sampler (scheduler) type / サンプラー（スケジューラ）の種類",
+    )
+    parser.add_argument(
+        "--scale",
+        type=float,
+        default=7.5,
+        help="unconditional guidance scale: eps = eps(x, empty) + scale * (eps(x, cond) - eps(x, empty)) / guidance scale",
+    )
+    parser.add_argument("--ckpt", type=str, default=None, help="path to checkpoint of model / モデルのcheckpointファイルまたはディレクトリ")
+    parser.add_argument(
+        "--vae", type=str, default=None, help="path to checkpoint of vae to replace / VAEを入れ替える場合、VAEのcheckpointファイルまたはディレクトリ"
+    )
+    parser.add_argument(
+        "--tokenizer_cache_dir",
+        type=str,
+        default=None,
+        help="directory for caching Tokenizer (for offline training) / Tokenizerをキャッシュするディレクトリ（ネット接続なしでの学習のため）",
+    )
+    # parser.add_argument("--replace_clip_l14_336", action='store_true',
+    #                     help="Replace CLIP (Text Encoder) to l/14@336 / CLIP(Text Encoder)をl/14@336に入れ替える")
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=None,
+        help="seed, or seed of seeds in multiple generation / 1枚生成時のseed、または複数枚生成時の乱数seedを決めるためのseed",
+    )
+    parser.add_argument(
+        "--iter_same_seed",
+        action="store_true",
+        help="use same seed for all prompts in iteration if no seed specified / 乱数seedの指定がないとき繰り返し内はすべて同じseedを使う（プロンプト間の差異の比較用）",
+    )
+    parser.add_argument("--fp16", action="store_true", help="use fp16 / fp16を指定し省メモリ化する")
+    parser.add_argument("--bf16", action="store_true", help="use bfloat16 / bfloat16を指定し省メモリ化する")
+    parser.add_argument("--xformers", action="store_true", help="use xformers / xformersを使用し高速化する")
+    parser.add_argument("--sdpa", action="store_true", help="use sdpa in PyTorch 2 / sdpa")
+    parser.add_argument(
+        "--diffusers_xformers",
+        action="store_true",
+        help="use xformers by diffusers (Hypernetworks doesn't work) / Diffusersでxformersを使用する（Hypernetwork利用不可）",
+    )
+    parser.add_argument(
+        "--opt_channels_last", action="store_true", help="set channels last option to model / モデルにchannels lastを指定し最適化する"
+    )
+    parser.add_argument(
+        "--network_module", type=str, default=None, nargs="*", help="additional network module to use / 追加ネットワークを使う時そのモジュール名"
+    )
+    parser.add_argument(
+        "--network_weights", type=str, default=None, nargs="*", help="additional network weights to load / 追加ネットワークの重み"
+    )
+    parser.add_argument("--network_mul", type=float, default=None, nargs="*", help="additional network multiplier / 追加ネットワークの効果の倍率")
+    parser.add_argument(
+        "--network_args", type=str, default=None, nargs="*", help="additional argmuments for network (key=value) / ネットワークへの追加の引数"
+    )
+    parser.add_argument("--network_show_meta", action="store_true", help="show metadata of network model / ネットワークモデルのメタデータを表示する")
+    parser.add_argument("--network_merge", action="store_true", help="merge network weights to original model / ネットワークの重みをマージする")
+    parser.add_argument(
+        "--network_pre_calc", action="store_true", help="pre-calculate network for generation / ネットワークのあらかじめ計算して生成する"
+    )
+    parser.add_argument(
+        "--textual_inversion_embeddings",
+        type=str,
+        default=None,
+        nargs="*",
+        help="Embeddings files of Textual Inversion / Textual Inversionのembeddings",
+    )
+    parser.add_argument(
+        "--XTI_embeddings",
+        type=str,
+        default=None,
+        nargs="*",
+        help="Embeddings files of Extended Textual Inversion / Extended Textual Inversionのembeddings",
+    )
+    parser.add_argument("--clip_skip", type=int, default=None, help="layer number from bottom to use in CLIP / CLIPの後ろからn層目の出力を使う")
+    parser.add_argument(
+        "--max_embeddings_multiples",
+        type=int,
+        default=None,
+        help="max embeding multiples, max token length is 75 * multiples / トークン長をデフォルトの何倍とするか 75*この値 がトークン長となる",
+    )
+    parser.add_argument(
+        "--clip_guidance_scale",
+        type=float,
+        default=0.0,
+        help="enable CLIP guided SD, scale for guidance (DDIM, PNDM, LMS samplers only) / CLIP guided SDを有効にしてこのscaleを適用する（サンプラーはDDIM、PNDM、LMSのみ）",
+    )
+    parser.add_argument(
+        "--clip_image_guidance_scale",
+        type=float,
+        default=0.0,
+        help="enable CLIP guided SD by image, scale for guidance / 画像によるCLIP guided SDを有効にしてこのscaleを適用する",
+    )
+    parser.add_argument(
+        "--vgg16_guidance_scale",
+        type=float,
+        default=0.0,
+        help="enable VGG16 guided SD by image, scale for guidance / 画像によるVGG16 guided SDを有効にしてこのscaleを適用する",
+    )
+    parser.add_argument(
+        "--vgg16_guidance_layer",
+        type=int,
+        default=20,
+        help="layer of VGG16 to calculate contents guide (1~30, 20 for conv4_2) / VGG16のcontents guideに使うレイヤー番号 (1~30、20はconv4_2)",
+    )
+    parser.add_argument(
+        "--guide_image_path", type=str, default=None, nargs="*", help="image to CLIP guidance / CLIP guided SDでガイドに使う画像"
+    )
+    parser.add_argument(
+        "--highres_fix_scale",
+        type=float,
+        default=None,
+        help="enable highres fix, reso scale for 1st stage / highres fixを有効にして最初の解像度をこのscaleにする",
+    )
+    parser.add_argument(
+        "--highres_fix_steps", type=int, default=28, help="1st stage steps for highres fix / highres fixの最初のステージのステップ数"
+    )
+    parser.add_argument(
+        "--highres_fix_strength",
+        type=float,
+        default=None,
+        help="1st stage img2img strength for highres fix / highres fixの最初のステージのimg2img時のstrength、省略時はstrengthと同じ",
+    )
+    parser.add_argument(
+        "--highres_fix_save_1st", action="store_true", help="save 1st stage images for highres fix / highres fixの最初のステージの画像を保存する"
+    )
+    parser.add_argument(
+        "--highres_fix_latents_upscaling",
+        action="store_true",
+        help="use latents upscaling for highres fix / highres fixでlatentで拡大する",
+    )
+    parser.add_argument(
+        "--highres_fix_upscaler", type=str, default=None, help="upscaler module for highres fix / highres fixで使うupscalerのモジュール名"
+    )
+    parser.add_argument(
+        "--highres_fix_upscaler_args",
+        type=str,
+        default=None,
+        help="additional argmuments for upscaler (key=value) / upscalerへの追加の引数",
+    )
+    parser.add_argument(
+        "--highres_fix_disable_control_net",
+        action="store_true",
+        help="disable ControlNet for highres fix / highres fixでControlNetを使わない",
+    )
+
+    parser.add_argument(
+        "--negative_scale", type=float, default=None, help="set another guidance scale for negative prompt / ネガティブプロンプトのscaleを指定する"
+    )
+
+    parser.add_argument(
+        "--control_net_models", type=str, default=None, nargs="*", help="ControlNet models to use / 使用するControlNetのモデル名"
+    )
+    parser.add_argument(
+        "--control_net_preps", type=str, default=None, nargs="*", help="ControlNet preprocess to use / 使用するControlNetのプリプロセス名"
+    )
+    parser.add_argument("--control_net_weights", type=float, default=None, nargs="*", help="ControlNet weights / ControlNetの重み")
+    parser.add_argument(
+        "--control_net_ratios",
+        type=float,
+        default=None,
+        nargs="*",
+        help="ControlNet guidance ratio for steps / ControlNetでガイドするステップ比率",
+    )
+    # parser.add_argument(
+    #     "--control_net_image_path", type=str, default=None, nargs="*", help="image for ControlNet guidance / ControlNetでガイドに使う画像"
+    # )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    main(args)
diff --git a/gui.bat b/gui.bat
new file mode 100644
index 0000000000000000000000000000000000000000..43b7d926a9ac01f525a134e457d782c1d59968d6
--- /dev/null
+++ b/gui.bat
@@ -0,0 +1,26 @@
+@echo off
+
+:: Deactivate the virtual environment
+call .\venv\Scripts\deactivate.bat
+
+:: Calling external python program to check for local modules
+python .\setup\check_local_modules.py --no_question
+
+:: Activate the virtual environment
+call .\venv\Scripts\activate.bat
+set PATH=%PATH%;%~dp0venv\Lib\site-packages\torch\lib
+
+:: Validate requirements
+python.exe .\setup\validate_requirements.py
+
+:: If the exit code is 0, run the kohya_gui.py script with the command-line arguments
+if %errorlevel% equ 0 (
+    REM Check if the batch was started via double-click
+    IF /i "%comspec% /c %~0 " equ "%cmdcmdline:"=%" (
+        REM echo This script was started by double clicking.
+        cmd /k python.exe kohya_gui.py %*
+    ) ELSE (
+        REM echo This script was started from a command prompt.
+        python.exe kohya_gui.py %*
+    )
+)
diff --git a/gui.ps1 b/gui.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..2947343db926b9a5a44a1e8e7e1a98fbf9d01ce5
--- /dev/null
+++ b/gui.ps1
@@ -0,0 +1,47 @@
+# Check if a virtual environment is active and deactivate it if necessary
+if ($env:VIRTUAL_ENV) {
+    # Write-Host "Deactivating the virtual environment to test for modules installed locally..."
+    & deactivate
+}
+
+# Run pip freeze and capture the output
+$pipOutput = & pip freeze
+
+# Check if modules are found in the output
+if ($pipOutput) {
+    Write-Host " "
+    Write-Host -ForegroundColor Yellow -Object "============================================================="
+    Write-Host -ForegroundColor Yellow -Object "Modules installed outside the virtual environment were found."
+    Write-Host -ForegroundColor Yellow -Object "This can cause issues. Please review the installed modules."
+    Write-Host " "
+    Write-Host -ForegroundColor Yellow -Object "You can deinstall all the local modules with:"
+    Write-Host " "
+    Write-Host -ForegroundColor Blue -Object "deactivate"
+    Write-Host -ForegroundColor Blue -Object "pip freeze > uninstall.txt"
+    Write-Host -ForegroundColor Blue -Object "pip uninstall -y -r uninstall.txt"
+    Write-Host -ForegroundColor Yellow -Object "============================================================="
+    Write-Host " "
+} 
+
+# Activate the virtual environment
+# Write-Host "Activating the virtual environment..."
+& .\venv\Scripts\activate
+$env:PATH += ";$($MyInvocation.MyCommand.Path)\venv\Lib\site-packages\torch\lib"
+
+# Debug info about system
+# python.exe .\setup\debug_info.py
+
+# Validate the requirements and store the exit code
+python.exe .\setup\validate_requirements.py
+
+# If the exit code is 0, read arguments from gui_parameters.txt (if it exists)
+# and run the kohya_gui.py script with the command-line arguments
+if ($LASTEXITCODE -eq 0) {
+    $argsFromFile = @()
+    if (Test-Path .\gui_parameters.txt) {
+        $argsFromFile = Get-Content .\gui_parameters.txt -Encoding UTF8 | Where-Object { $_ -notmatch "^#" } | Foreach-Object { $_ -split " " }
+    }
+    $args_combo = $argsFromFile + $args
+    # Write-Host "The arguments passed to this script were: $args_combo"
+    python.exe kohya_gui.py $args_combo
+}
diff --git a/gui.sh b/gui.sh
new file mode 100644
index 0000000000000000000000000000000000000000..7f66a3effbcdc00be1bb6ca2adb9dfbbaa0e64e6
--- /dev/null
+++ b/gui.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Checks to see if variable is set and non-empty.
+# This is defined first, so we can use the function for some default variable values
+env_var_exists() {
+  if [[ -n "${!1}" ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# Need RUNPOD to have a default value before first access
+RUNPOD=false
+if env_var_exists RUNPOD_POD_ID || env_var_exists RUNPOD_API_KEY; then
+  RUNPOD=true
+fi
+
+# If it is run with the sudo command, get the complete LD_LIBRARY_PATH environment variable of the system and assign it to the current environment,
+# because it will be used later.
+if [ -n "$SUDO_USER" ] || [ -n "$SUDO_COMMAND" ]; then
+    echo "The sudo command resets the non-essential environment variables, we keep the LD_LIBRARY_PATH variable."
+    export LD_LIBRARY_PATH=$(sudo -i printenv LD_LIBRARY_PATH)
+fi
+
+# This gets the directory the script is run from so pathing can work relative to the script where needed.
+SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd)
+
+# Step into GUI local directory
+cd "$SCRIPT_DIR" || exit 1
+
+if [ -d "$SCRIPT_DIR/venv" ]; then
+    source "$SCRIPT_DIR/venv/bin/activate" || exit 1
+else
+    echo "venv folder does not exist. Not activating..."
+fi
+
+# Check if LD_LIBRARY_PATH environment variable exists
+if [[ -z "${LD_LIBRARY_PATH}" ]]; then
+    # Set the ANSI escape sequence for yellow text
+    YELLOW='\033[0;33m'
+    # Set the ANSI escape sequence to reset text color
+    RESET='\033[0m'
+    
+    echo -e "${YELLOW}Warning: LD_LIBRARY_PATH environment variable is not set.${RESET}"
+    echo -e "${YELLOW}Certain functionalities may not work correctly.${RESET}"
+    echo -e "${YELLOW}Please ensure that the required libraries are properly configured.${RESET}"
+    echo -e " "
+    echo -e "${YELLOW}If you use WSL2 you may want to: export LD_LIBRARY_PATH=/usr/lib/wsl/lib/${RESET}"
+    echo -e " "
+fi
+
+# Determine the requirements file based on the system
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    if [[ "$(uname -m)" == "arm64" ]]; then
+        REQUIREMENTS_FILE="$SCRIPT_DIR/requirements_macos_arm64.txt"
+    else
+        REQUIREMENTS_FILE="$SCRIPT_DIR/requirements_macos_amd64.txt"
+    fi
+else
+    if [ "$RUNPOD" = false ]; then
+        REQUIREMENTS_FILE="$SCRIPT_DIR/requirements_linux.txt"
+    else
+        REQUIREMENTS_FILE="$SCRIPT_DIR/requirements_runpod.txt"
+    fi
+fi
+
+# Validate the requirements and run the script if successful
+if python "$SCRIPT_DIR/setup/validate_requirements.py" -r "$REQUIREMENTS_FILE"; then
+    python "$SCRIPT_DIR/kohya_gui.py" "$@"
+fi
diff --git a/kohya_gui.py b/kohya_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..a6043e4b922f6b08ee6992dd8b3086260d080714
--- /dev/null
+++ b/kohya_gui.py
@@ -0,0 +1,141 @@
+import gradio as gr
+import os
+import argparse
+from dreambooth_gui import dreambooth_tab
+from finetune_gui import finetune_tab
+from textual_inversion_gui import ti_tab
+from library.utilities import utilities_tab
+from lora_gui import lora_tab
+from library.class_lora_tab import LoRATools
+
+import os
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+
+def UI(**kwargs):
+    css = ''
+
+    headless = kwargs.get('headless', False)
+    log.info(f'headless: {headless}')
+
+    if os.path.exists('./style.css'):
+        with open(os.path.join('./style.css'), 'r', encoding='utf8') as file:
+            log.info('Load CSS...')
+            css += file.read() + '\n'
+
+    if os.path.exists('./.release'):
+        with open(os.path.join('./.release'), 'r', encoding='utf8') as file:
+            release = file.read()
+
+    if os.path.exists('./README.md'):
+        with open(os.path.join('./README.md'), 'r', encoding='utf8') as file:
+            README = file.read()
+
+    interface = gr.Blocks(
+        css=css, title=f'Kohya_ss GUI {release}', theme=gr.themes.Default()
+    )
+
+    with interface:
+        with gr.Tab('Dreambooth'):
+            (
+                train_data_dir_input,
+                reg_data_dir_input,
+                output_dir_input,
+                logging_dir_input,
+            ) = dreambooth_tab(headless=headless)
+        with gr.Tab('LoRA'):
+            lora_tab(headless=headless)
+        with gr.Tab('Textual Inversion'):
+            ti_tab(headless=headless)
+        with gr.Tab('Finetuning'):
+            finetune_tab(headless=headless)
+        with gr.Tab('Utilities'):
+            utilities_tab(
+                train_data_dir_input=train_data_dir_input,
+                reg_data_dir_input=reg_data_dir_input,
+                output_dir_input=output_dir_input,
+                logging_dir_input=logging_dir_input,
+                enable_copy_info_button=True,
+                headless=headless,
+            )
+            with gr.Tab('LoRA'):
+                _ = LoRATools(headless=headless)
+        with gr.Tab('About'):
+            gr.Markdown(f'kohya_ss GUI release {release}')
+            with gr.Tab('README'):
+                gr.Markdown(README)
+
+        htmlStr = f"""
+        <html>
+            <body>
+                <div class="ver-class">{release}</div>
+            </body>
+        </html>
+        """
+        gr.HTML(htmlStr)
+    # Show the interface
+    launch_kwargs = {}
+    username = kwargs.get('username')
+    password = kwargs.get('password')
+    server_port = kwargs.get('server_port', 0)
+    inbrowser = kwargs.get('inbrowser', False)
+    share = kwargs.get('share', False)
+    server_name = kwargs.get('listen')
+
+    launch_kwargs['server_name'] = server_name
+    if username and password:
+        launch_kwargs['auth'] = (username, password)
+    if server_port > 0:
+        launch_kwargs['server_port'] = server_port
+    if inbrowser:
+        launch_kwargs['inbrowser'] = inbrowser
+    if share:
+        launch_kwargs['share'] = share
+    interface.launch(**launch_kwargs)
+
+
+if __name__ == '__main__':
+    # torch.cuda.set_per_process_memory_fraction(0.48)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--listen',
+        type=str,
+        default='127.0.0.1',
+        help='IP to listen on for connections to Gradio',
+    )
+    parser.add_argument(
+        '--username', type=str, default='', help='Username for authentication'
+    )
+    parser.add_argument(
+        '--password', type=str, default='', help='Password for authentication'
+    )
+    parser.add_argument(
+        '--server_port',
+        type=int,
+        default=0,
+        help='Port to run the server listener on',
+    )
+    parser.add_argument(
+        '--inbrowser', action='store_true', help='Open in browser'
+    )
+    parser.add_argument(
+        '--share', action='store_true', help='Share the gradio UI'
+    )
+    parser.add_argument(
+        '--headless', action='store_true', help='Is the server headless'
+    )
+
+    args = parser.parse_args()
+
+    UI(
+        username=args.username,
+        password=args.password,
+        inbrowser=args.inbrowser,
+        server_port=args.server_port,
+        share=args.share,
+        listen=args.listen,
+        headless=args.headless,
+    )
diff --git a/library/__init__.py b/library/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/library/__pycache__/__init__.cpython-310.pyc b/library/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9155294611d41f8ec448bf676dfb460c5d495698
Binary files /dev/null and b/library/__pycache__/__init__.cpython-310.pyc differ
diff --git a/library/__pycache__/attention_processors.cpython-310.pyc b/library/__pycache__/attention_processors.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7ae545e66faf50943369df93803935c953390e51
Binary files /dev/null and b/library/__pycache__/attention_processors.cpython-310.pyc differ
diff --git a/library/__pycache__/basic_caption_gui.cpython-310.pyc b/library/__pycache__/basic_caption_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..922d904eb967221f7de3f5d5c3a81a972049b90b
Binary files /dev/null and b/library/__pycache__/basic_caption_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/blip_caption_gui.cpython-310.pyc b/library/__pycache__/blip_caption_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..08dbd6629b9b66323147157ee21d245bd6be8b1d
Binary files /dev/null and b/library/__pycache__/blip_caption_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/class_advanced_training.cpython-310.pyc b/library/__pycache__/class_advanced_training.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4a773c452d7c3c25f3b52680239cb82c2eb51938
Binary files /dev/null and b/library/__pycache__/class_advanced_training.cpython-310.pyc differ
diff --git a/library/__pycache__/class_basic_training.cpython-310.pyc b/library/__pycache__/class_basic_training.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..219552a03c82ecdabca93944d309084a8af4e99f
Binary files /dev/null and b/library/__pycache__/class_basic_training.cpython-310.pyc differ
diff --git a/library/__pycache__/class_configuration_file.cpython-310.pyc b/library/__pycache__/class_configuration_file.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fb712343f1ca95dee3012ec09b34e7456b2a25ba
Binary files /dev/null and b/library/__pycache__/class_configuration_file.cpython-310.pyc differ
diff --git a/library/__pycache__/class_folders.cpython-310.pyc b/library/__pycache__/class_folders.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..127914f9a735098ef3a6350d9f1596f0f4cb740b
Binary files /dev/null and b/library/__pycache__/class_folders.cpython-310.pyc differ
diff --git a/library/__pycache__/class_lora_tab.cpython-310.pyc b/library/__pycache__/class_lora_tab.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..eb090a0fdf465d09d837eaea8f02e0989e0e366b
Binary files /dev/null and b/library/__pycache__/class_lora_tab.cpython-310.pyc differ
diff --git a/library/__pycache__/class_sample_images.cpython-310.pyc b/library/__pycache__/class_sample_images.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..180ec525087401eac8f6d5939cb78ebc4c817858
Binary files /dev/null and b/library/__pycache__/class_sample_images.cpython-310.pyc differ
diff --git a/library/__pycache__/class_sdxl_parameters.cpython-310.pyc b/library/__pycache__/class_sdxl_parameters.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1c1da5189894065c5f653c6f6bd70dc9089f0c5b
Binary files /dev/null and b/library/__pycache__/class_sdxl_parameters.cpython-310.pyc differ
diff --git a/library/__pycache__/class_source_model.cpython-310.pyc b/library/__pycache__/class_source_model.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..15c7b409369f6a9f490453f59b8cb72bc2f95752
Binary files /dev/null and b/library/__pycache__/class_source_model.cpython-310.pyc differ
diff --git a/library/__pycache__/common_gui.cpython-310.pyc b/library/__pycache__/common_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3859f0c6e24058fb7626970d1900d69e00649033
Binary files /dev/null and b/library/__pycache__/common_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/config_util.cpython-310.pyc b/library/__pycache__/config_util.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8ec6c538c0907bb37b8c026aa7e63b2335d21c55
Binary files /dev/null and b/library/__pycache__/config_util.cpython-310.pyc differ
diff --git a/library/__pycache__/convert_model_gui.cpython-310.pyc b/library/__pycache__/convert_model_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..903e89662d154597449b7e8e8d2f7113efc6a078
Binary files /dev/null and b/library/__pycache__/convert_model_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/custom_logging.cpython-310.pyc b/library/__pycache__/custom_logging.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..298e752c7f0f18db7d4f1b890de8373aad5bcbe7
Binary files /dev/null and b/library/__pycache__/custom_logging.cpython-310.pyc differ
diff --git a/library/__pycache__/custom_train_functions.cpython-310.pyc b/library/__pycache__/custom_train_functions.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..087f3ee79ff517fdd564fe2a0336f4ecec3d87e7
Binary files /dev/null and b/library/__pycache__/custom_train_functions.cpython-310.pyc differ
diff --git a/library/__pycache__/dataset_balancing_gui.cpython-310.pyc b/library/__pycache__/dataset_balancing_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..00b8062f44fdd1b8438b0caccf34d5dc6b55b686
Binary files /dev/null and b/library/__pycache__/dataset_balancing_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/dreambooth_folder_creation_gui.cpython-310.pyc b/library/__pycache__/dreambooth_folder_creation_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d95b2cfd874064102d0b4cbffbb20d170dc4f0a0
Binary files /dev/null and b/library/__pycache__/dreambooth_folder_creation_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/extract_lora_from_dylora_gui.cpython-310.pyc b/library/__pycache__/extract_lora_from_dylora_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8db51dd27efca9abf98ab119262f55ce4cb7d072
Binary files /dev/null and b/library/__pycache__/extract_lora_from_dylora_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/extract_lora_gui.cpython-310.pyc b/library/__pycache__/extract_lora_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9f20b248cd8d4f7684edbd7fb06bb3fc46644345
Binary files /dev/null and b/library/__pycache__/extract_lora_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/extract_lycoris_locon_gui.cpython-310.pyc b/library/__pycache__/extract_lycoris_locon_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bad36a11c5103f9c4e8f852ee5108ca2d685ae5d
Binary files /dev/null and b/library/__pycache__/extract_lycoris_locon_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/git_caption_gui.cpython-310.pyc b/library/__pycache__/git_caption_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5643ee3adfe08d0e377c953bad26ac875747b70c
Binary files /dev/null and b/library/__pycache__/git_caption_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/group_images_gui.cpython-310.pyc b/library/__pycache__/group_images_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..568685c869b51902392253dee2bd36db0f610b71
Binary files /dev/null and b/library/__pycache__/group_images_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/huggingface_util.cpython-310.pyc b/library/__pycache__/huggingface_util.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..224fbd20aaca35f79e684f2afa8fc54573be7c1e
Binary files /dev/null and b/library/__pycache__/huggingface_util.cpython-310.pyc differ
diff --git a/library/__pycache__/hypernetwork.cpython-310.pyc b/library/__pycache__/hypernetwork.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2f0b57a8b92cf431b6e6580c358860fb940a2b0c
Binary files /dev/null and b/library/__pycache__/hypernetwork.cpython-310.pyc differ
diff --git a/library/__pycache__/lpw_stable_diffusion.cpython-310.pyc b/library/__pycache__/lpw_stable_diffusion.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..39da3fae6fcc7e02b8ccd1ef5da900979db35325
Binary files /dev/null and b/library/__pycache__/lpw_stable_diffusion.cpython-310.pyc differ
diff --git a/library/__pycache__/merge_lora_gui.cpython-310.pyc b/library/__pycache__/merge_lora_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6a60726ba6c7c9fbefa49c7beee62e0f7007a5e1
Binary files /dev/null and b/library/__pycache__/merge_lora_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/merge_lycoris_gui.cpython-310.pyc b/library/__pycache__/merge_lycoris_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e3509279f198f6e1a8867db95afa8db0eff01b37
Binary files /dev/null and b/library/__pycache__/merge_lycoris_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/model_util.cpython-310.pyc b/library/__pycache__/model_util.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f8c027c253070176cff5cf9d98955818ccc37cb8
Binary files /dev/null and b/library/__pycache__/model_util.cpython-310.pyc differ
diff --git a/library/__pycache__/original_unet.cpython-310.pyc b/library/__pycache__/original_unet.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ec78270eb8534a99d1ce484fbde2c7a6e78dfe43
Binary files /dev/null and b/library/__pycache__/original_unet.cpython-310.pyc differ
diff --git a/library/__pycache__/resize_lora_gui.cpython-310.pyc b/library/__pycache__/resize_lora_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4c005166c10cb26586ec1ee1128eeb9420cf1b21
Binary files /dev/null and b/library/__pycache__/resize_lora_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/svd_merge_lora_gui.cpython-310.pyc b/library/__pycache__/svd_merge_lora_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bd85ac4afa81d83edbfe73c0d14ee8b4641f1271
Binary files /dev/null and b/library/__pycache__/svd_merge_lora_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/tensorboard_gui.cpython-310.pyc b/library/__pycache__/tensorboard_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..128b8ca5dc190d041eef319a057d046d00654fdb
Binary files /dev/null and b/library/__pycache__/tensorboard_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/train_util.cpython-310.pyc b/library/__pycache__/train_util.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e6b51a5adb27c46cc22984925f61c98844aec55f
Binary files /dev/null and b/library/__pycache__/train_util.cpython-310.pyc differ
diff --git a/library/__pycache__/utilities.cpython-310.pyc b/library/__pycache__/utilities.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d3f5453f820f5b232878ec622c79fa8f1b031358
Binary files /dev/null and b/library/__pycache__/utilities.cpython-310.pyc differ
diff --git a/library/__pycache__/utils.cpython-310.pyc b/library/__pycache__/utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7dde59e08c1827f3cadfc18122f0ce0a28545887
Binary files /dev/null and b/library/__pycache__/utils.cpython-310.pyc differ
diff --git a/library/__pycache__/verify_lora_gui.cpython-310.pyc b/library/__pycache__/verify_lora_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d91ba54145d2461fe585f5313702a7888ffcc9cc
Binary files /dev/null and b/library/__pycache__/verify_lora_gui.cpython-310.pyc differ
diff --git a/library/__pycache__/wd14_caption_gui.cpython-310.pyc b/library/__pycache__/wd14_caption_gui.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3ea9133bf9d8e8733a887d48a44722cc2f52e66a
Binary files /dev/null and b/library/__pycache__/wd14_caption_gui.cpython-310.pyc differ
diff --git a/library/attention_processors.py b/library/attention_processors.py
new file mode 100644
index 0000000000000000000000000000000000000000..310c2cb1c63955f8f03296c54fd47c21f1a981c9
--- /dev/null
+++ b/library/attention_processors.py
@@ -0,0 +1,227 @@
+import math
+from typing import Any
+from einops import rearrange
+import torch
+from diffusers.models.attention_processor import Attention
+
+
+# flash attention forwards and backwards
+
+# https://arxiv.org/abs/2205.14135
+
+EPSILON = 1e-6
+
+
+class FlashAttentionFunction(torch.autograd.function.Function):
+    @staticmethod
+    @torch.no_grad()
+    def forward(ctx, q, k, v, mask, causal, q_bucket_size, k_bucket_size):
+        """Algorithm 2 in the paper"""
+
+        device = q.device
+        dtype = q.dtype
+        max_neg_value = -torch.finfo(q.dtype).max
+        qk_len_diff = max(k.shape[-2] - q.shape[-2], 0)
+
+        o = torch.zeros_like(q)
+        all_row_sums = torch.zeros((*q.shape[:-1], 1), dtype=dtype, device=device)
+        all_row_maxes = torch.full(
+            (*q.shape[:-1], 1), max_neg_value, dtype=dtype, device=device
+        )
+
+        scale = q.shape[-1] ** -0.5
+
+        if mask is None:
+            mask = (None,) * math.ceil(q.shape[-2] / q_bucket_size)
+        else:
+            mask = rearrange(mask, "b n -> b 1 1 n")
+            mask = mask.split(q_bucket_size, dim=-1)
+
+        row_splits = zip(
+            q.split(q_bucket_size, dim=-2),
+            o.split(q_bucket_size, dim=-2),
+            mask,
+            all_row_sums.split(q_bucket_size, dim=-2),
+            all_row_maxes.split(q_bucket_size, dim=-2),
+        )
+
+        for ind, (qc, oc, row_mask, row_sums, row_maxes) in enumerate(row_splits):
+            q_start_index = ind * q_bucket_size - qk_len_diff
+
+            col_splits = zip(
+                k.split(k_bucket_size, dim=-2),
+                v.split(k_bucket_size, dim=-2),
+            )
+
+            for k_ind, (kc, vc) in enumerate(col_splits):
+                k_start_index = k_ind * k_bucket_size
+
+                attn_weights = (
+                    torch.einsum("... i d, ... j d -> ... i j", qc, kc) * scale
+                )
+
+                if row_mask is not None:
+                    attn_weights.masked_fill_(~row_mask, max_neg_value)
+
+                if causal and q_start_index < (k_start_index + k_bucket_size - 1):
+                    causal_mask = torch.ones(
+                        (qc.shape[-2], kc.shape[-2]), dtype=torch.bool, device=device
+                    ).triu(q_start_index - k_start_index + 1)
+                    attn_weights.masked_fill_(causal_mask, max_neg_value)
+
+                block_row_maxes = attn_weights.amax(dim=-1, keepdims=True)
+                attn_weights -= block_row_maxes
+                exp_weights = torch.exp(attn_weights)
+
+                if row_mask is not None:
+                    exp_weights.masked_fill_(~row_mask, 0.0)
+
+                block_row_sums = exp_weights.sum(dim=-1, keepdims=True).clamp(
+                    min=EPSILON
+                )
+
+                new_row_maxes = torch.maximum(block_row_maxes, row_maxes)
+
+                exp_values = torch.einsum(
+                    "... i j, ... j d -> ... i d", exp_weights, vc
+                )
+
+                exp_row_max_diff = torch.exp(row_maxes - new_row_maxes)
+                exp_block_row_max_diff = torch.exp(block_row_maxes - new_row_maxes)
+
+                new_row_sums = (
+                    exp_row_max_diff * row_sums
+                    + exp_block_row_max_diff * block_row_sums
+                )
+
+                oc.mul_((row_sums / new_row_sums) * exp_row_max_diff).add_(
+                    (exp_block_row_max_diff / new_row_sums) * exp_values
+                )
+
+                row_maxes.copy_(new_row_maxes)
+                row_sums.copy_(new_row_sums)
+
+        ctx.args = (causal, scale, mask, q_bucket_size, k_bucket_size)
+        ctx.save_for_backward(q, k, v, o, all_row_sums, all_row_maxes)
+
+        return o
+
+    @staticmethod
+    @torch.no_grad()
+    def backward(ctx, do):
+        """Algorithm 4 in the paper"""
+
+        causal, scale, mask, q_bucket_size, k_bucket_size = ctx.args
+        q, k, v, o, l, m = ctx.saved_tensors
+
+        device = q.device
+
+        max_neg_value = -torch.finfo(q.dtype).max
+        qk_len_diff = max(k.shape[-2] - q.shape[-2], 0)
+
+        dq = torch.zeros_like(q)
+        dk = torch.zeros_like(k)
+        dv = torch.zeros_like(v)
+
+        row_splits = zip(
+            q.split(q_bucket_size, dim=-2),
+            o.split(q_bucket_size, dim=-2),
+            do.split(q_bucket_size, dim=-2),
+            mask,
+            l.split(q_bucket_size, dim=-2),
+            m.split(q_bucket_size, dim=-2),
+            dq.split(q_bucket_size, dim=-2),
+        )
+
+        for ind, (qc, oc, doc, row_mask, lc, mc, dqc) in enumerate(row_splits):
+            q_start_index = ind * q_bucket_size - qk_len_diff
+
+            col_splits = zip(
+                k.split(k_bucket_size, dim=-2),
+                v.split(k_bucket_size, dim=-2),
+                dk.split(k_bucket_size, dim=-2),
+                dv.split(k_bucket_size, dim=-2),
+            )
+
+            for k_ind, (kc, vc, dkc, dvc) in enumerate(col_splits):
+                k_start_index = k_ind * k_bucket_size
+
+                attn_weights = (
+                    torch.einsum("... i d, ... j d -> ... i j", qc, kc) * scale
+                )
+
+                if causal and q_start_index < (k_start_index + k_bucket_size - 1):
+                    causal_mask = torch.ones(
+                        (qc.shape[-2], kc.shape[-2]), dtype=torch.bool, device=device
+                    ).triu(q_start_index - k_start_index + 1)
+                    attn_weights.masked_fill_(causal_mask, max_neg_value)
+
+                exp_attn_weights = torch.exp(attn_weights - mc)
+
+                if row_mask is not None:
+                    exp_attn_weights.masked_fill_(~row_mask, 0.0)
+
+                p = exp_attn_weights / lc
+
+                dv_chunk = torch.einsum("... i j, ... i d -> ... j d", p, doc)
+                dp = torch.einsum("... i d, ... j d -> ... i j", doc, vc)
+
+                D = (doc * oc).sum(dim=-1, keepdims=True)
+                ds = p * scale * (dp - D)
+
+                dq_chunk = torch.einsum("... i j, ... j d -> ... i d", ds, kc)
+                dk_chunk = torch.einsum("... i j, ... i d -> ... j d", ds, qc)
+
+                dqc.add_(dq_chunk)
+                dkc.add_(dk_chunk)
+                dvc.add_(dv_chunk)
+
+        return dq, dk, dv, None, None, None, None
+
+
+class FlashAttnProcessor:
+    def __call__(
+        self,
+        attn: Attention,
+        hidden_states,
+        encoder_hidden_states=None,
+        attention_mask=None,
+    ) -> Any:
+        q_bucket_size = 512
+        k_bucket_size = 1024
+
+        h = attn.heads
+        q = attn.to_q(hidden_states)
+
+        encoder_hidden_states = (
+            encoder_hidden_states
+            if encoder_hidden_states is not None
+            else hidden_states
+        )
+        encoder_hidden_states = encoder_hidden_states.to(hidden_states.dtype)
+
+        if hasattr(attn, "hypernetwork") and attn.hypernetwork is not None:
+            context_k, context_v = attn.hypernetwork.forward(
+                hidden_states, encoder_hidden_states
+            )
+            context_k = context_k.to(hidden_states.dtype)
+            context_v = context_v.to(hidden_states.dtype)
+        else:
+            context_k = encoder_hidden_states
+            context_v = encoder_hidden_states
+
+        k = attn.to_k(context_k)
+        v = attn.to_v(context_v)
+        del encoder_hidden_states, hidden_states
+
+        q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=h), (q, k, v))
+
+        out = FlashAttentionFunction.apply(
+            q, k, v, attention_mask, False, q_bucket_size, k_bucket_size
+        )
+
+        out = rearrange(out, "b h n d -> b n (h d)")
+
+        out = attn.to_out[0](out)
+        out = attn.to_out[1](out)
+        return out
diff --git a/library/basic_caption_gui.py b/library/basic_caption_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..95619ac53c3620871c8c938c0d1253ff8ee6797d
--- /dev/null
+++ b/library/basic_caption_gui.py
@@ -0,0 +1,156 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+from .common_gui import get_folder_path, add_pre_postfix, find_replace
+import os
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+
+def caption_images(
+    caption_text,
+    images_dir,
+    overwrite,
+    caption_ext,
+    prefix,
+    postfix,
+    find_text,
+    replace_text,
+):
+    # Check if images_dir is provided
+    if not images_dir:
+        msgbox('Image folder is missing. Please provide the directory containing the images to caption.')
+        return
+
+    # Check if caption_ext is provided
+    if not caption_ext:
+        msgbox('Please provide an extension for the caption files.')
+        return
+
+    if caption_text:
+        log.info(f'Captioning files in {images_dir} with {caption_text}...')
+
+        # Build the command to run caption.py
+        run_cmd = f'python "tools/caption.py"'
+        run_cmd += f' --caption_text="{caption_text}"'
+
+        # Add optional flags to the command
+        if overwrite:
+            run_cmd += f' --overwrite'
+        if caption_ext:
+            run_cmd += f' --caption_file_ext="{caption_ext}"'
+
+        run_cmd += f' "{images_dir}"'
+
+        log.info(run_cmd)
+
+        # Run the command based on the operating system
+        if os.name == 'posix':
+            os.system(run_cmd)
+        else:
+            subprocess.run(run_cmd)
+
+    # Check if overwrite option is enabled
+    if overwrite:
+        if prefix or postfix:
+            # Add prefix and postfix to caption files
+            add_pre_postfix(
+                folder=images_dir,
+                caption_file_ext=caption_ext,
+                prefix=prefix,
+                postfix=postfix,
+            )
+        if find_text:
+            # Find and replace text in caption files
+            find_replace(
+                folder_path=images_dir,
+                caption_file_ext=caption_ext,
+                search_text=find_text,
+                replace_text=replace_text,
+            )
+    else:
+        if prefix or postfix:
+            # Show a message if modification is not possible without overwrite option enabled
+            msgbox(
+                'Could not modify caption files with requested change because the "Overwrite existing captions in folder" option is not selected.'
+            )
+
+    log.info('Captioning done.')
+
+
+# Gradio UI
+def gradio_basic_caption_gui_tab(headless=False):
+    with gr.Tab('Basic Captioning'):
+        gr.Markdown(
+            'This utility allows you to create simple caption files for each image in a folder.'
+        )
+        with gr.Row():
+            images_dir = gr.Textbox(
+                label='Image folder to caption',
+                placeholder='Directory containing the images to caption',
+                interactive=True,
+            )
+            folder_button = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            folder_button.click(
+                get_folder_path,
+                outputs=images_dir,
+                show_progress=False,
+            )
+            caption_ext = gr.Textbox(
+                label='Caption file extension',
+                placeholder='Extension for caption file (e.g., .caption, .txt)',
+                value='.txt',
+                interactive=True,
+            )
+            overwrite = gr.Checkbox(
+                label='Overwrite existing captions in folder',
+                interactive=True,
+                value=False,
+            )
+        with gr.Row():
+            prefix = gr.Textbox(
+                label='Prefix to add to caption',
+                placeholder='(Optional)',
+                interactive=True,
+            )
+            caption_text = gr.Textbox(
+                label='Caption text',
+                placeholder='e.g., "by some artist". Leave empty if you only want to add a prefix or postfix.',
+                interactive=True,
+            )
+            postfix = gr.Textbox(
+                label='Postfix to add to caption',
+                placeholder='(Optional)',
+                interactive=True,
+            )
+        with gr.Row():
+            find_text = gr.Textbox(
+                label='Find text',
+                placeholder='e.g., "by some artist". Leave empty if you only want to add a prefix or postfix.',
+                interactive=True,
+            )
+            replace_text = gr.Textbox(
+                label='Replacement text',
+                placeholder='e.g., "by some artist". Leave empty if you want to replace with nothing.',
+                interactive=True,
+            )
+            caption_button = gr.Button('Caption images')
+            caption_button.click(
+                caption_images,
+                inputs=[
+                    caption_text,
+                    images_dir,
+                    overwrite,
+                    caption_ext,
+                    prefix,
+                    postfix,
+                    find_text,
+                    replace_text,
+                ],
+                show_progress=False,
+            )
diff --git a/library/blip_caption_gui.py b/library/blip_caption_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..3679140c8671d47af44acf37f407ea1edecdb8af
--- /dev/null
+++ b/library/blip_caption_gui.py
@@ -0,0 +1,151 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import get_folder_path, add_pre_postfix
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def caption_images(
+    train_data_dir,
+    caption_file_ext,
+    batch_size,
+    num_beams,
+    top_p,
+    max_length,
+    min_length,
+    beam_search,
+    prefix,
+    postfix,
+):
+    # Check if the image folder is provided
+    if train_data_dir == '':
+        msgbox('Image folder is missing...')
+        return
+
+    # Check if the caption file extension is provided
+    if caption_file_ext == '':
+        msgbox('Please provide an extension for the caption files.')
+        return
+
+    log.info(f'Captioning files in {train_data_dir}...')
+
+    # Construct the command to run
+    run_cmd = f'{PYTHON} "finetune/make_captions.py"'
+    run_cmd += f' --batch_size="{int(batch_size)}"'
+    run_cmd += f' --num_beams="{int(num_beams)}"'
+    run_cmd += f' --top_p="{top_p}"'
+    run_cmd += f' --max_length="{int(max_length)}"'
+    run_cmd += f' --min_length="{int(min_length)}"'
+    if beam_search:
+        run_cmd += f' --beam_search'
+    if caption_file_ext != '':
+        run_cmd += f' --caption_extension="{caption_file_ext}"'
+    run_cmd += f' "{train_data_dir}"'
+    run_cmd += f' --caption_weights="https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_large_caption.pth"'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+    # Add prefix and postfix
+    add_pre_postfix(
+        folder=train_data_dir,
+        caption_file_ext=caption_file_ext,
+        prefix=prefix,
+        postfix=postfix,
+    )
+
+    log.info('...captioning done')
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_blip_caption_gui_tab(headless=False):
+    with gr.Tab('BLIP Captioning'):
+        gr.Markdown(
+            'This utility uses BLIP to caption files for each image in a folder.'
+        )
+        with gr.Row():
+            train_data_dir = gr.Textbox(
+                label='Image folder to caption',
+                placeholder='Directory containing the images to caption',
+                interactive=True,
+            )
+            button_train_data_dir_input = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            button_train_data_dir_input.click(
+                get_folder_path,
+                outputs=train_data_dir,
+                show_progress=False,
+            )
+        with gr.Row():
+            caption_file_ext = gr.Textbox(
+                label='Caption file extension',
+                placeholder='Extension for caption file, e.g., .caption, .txt',
+                value='.txt',
+                interactive=True,
+            )
+
+            prefix = gr.Textbox(
+                label='Prefix to add to BLIP caption',
+                placeholder='(Optional)',
+                interactive=True,
+            )
+
+            postfix = gr.Textbox(
+                label='Postfix to add to BLIP caption',
+                placeholder='(Optional)',
+                interactive=True,
+            )
+
+            batch_size = gr.Number(
+                value=1, label='Batch size', interactive=True
+            )
+
+        with gr.Row():
+            beam_search = gr.Checkbox(
+                label='Use beam search', interactive=True, value=True
+            )
+            num_beams = gr.Number(
+                value=1, label='Number of beams', interactive=True
+            )
+            top_p = gr.Number(value=0.9, label='Top p', interactive=True)
+            max_length = gr.Number(
+                value=75, label='Max length', interactive=True
+            )
+            min_length = gr.Number(
+                value=5, label='Min length', interactive=True
+            )
+
+        caption_button = gr.Button('Caption images')
+
+        caption_button.click(
+            caption_images,
+            inputs=[
+                train_data_dir,
+                caption_file_ext,
+                batch_size,
+                num_beams,
+                top_p,
+                max_length,
+                min_length,
+                beam_search,
+                prefix,
+                postfix,
+            ],
+            show_progress=False,
+        )
diff --git a/library/class_advanced_training.py b/library/class_advanced_training.py
new file mode 100644
index 0000000000000000000000000000000000000000..443eb41401d43275acd14ac7079ad4d7396a031d
--- /dev/null
+++ b/library/class_advanced_training.py
@@ -0,0 +1,247 @@
+import gradio as gr
+from .common_gui import get_folder_path, get_any_file_path
+
+class AdvancedTraining:
+    def __init__(
+        self,
+        headless=False,
+        finetuning: bool = False
+    ):
+        self.headless = headless
+        self.finetuning = finetuning
+        def noise_offset_type_change(noise_offset_type):
+            if noise_offset_type == 'Original':
+                return (gr.Group.update(visible=True), gr.Group.update(visible=False))
+            else:
+                return (gr.Group.update(visible=False), gr.Group.update(visible=True))
+
+        with gr.Row(visible=not finetuning):
+            self.no_token_padding = gr.Checkbox(
+                label='No token padding', value=False
+            )
+            self.gradient_accumulation_steps = gr.Number(
+                label='Gradient accumulate steps', value='1'
+            )
+            self.weighted_captions = gr.Checkbox(
+                label='Weighted captions', value=False
+            )
+        with gr.Row(visible=not finetuning):
+            self.prior_loss_weight = gr.Number(
+                label='Prior loss weight', value=1.0
+            )
+            self.vae = gr.Textbox(
+                label='VAE',
+                placeholder='(Optiona) path to checkpoint of vae to replace for training',
+            )
+            self.vae_button = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            self.vae_button.click(
+                get_any_file_path,
+                outputs=self.vae,
+                show_progress=False,
+            )
+        with gr.Row(visible=not finetuning):
+            self.lr_scheduler_num_cycles = gr.Textbox(
+                label='LR number of cycles',
+                placeholder='(Optional) For Cosine with restart and polynomial only',
+            )
+
+            self.lr_scheduler_power = gr.Textbox(
+                label='LR power',
+                placeholder='(Optional) For Cosine with restart and polynomial only',
+            )
+
+        with gr.Row():
+            self.additional_parameters = gr.Textbox(
+                label='Additional parameters',
+                placeholder='(Optional) Use to provide additional parameters not handled by the GUI. Eg: --some_parameters "value"',
+            )
+        with gr.Row():
+            self.save_every_n_steps = gr.Number(
+                label='Save every N steps',
+                value=0,
+                precision=0,
+                info='(Optional) The model is saved every specified steps',
+            )
+            self.save_last_n_steps = gr.Number(
+                label='Save last N steps',
+                value=0,
+                precision=0,
+                info='(Optional) Save only the specified number of models (old models will be deleted)',
+            )
+            self.save_last_n_steps_state = gr.Number(
+                label='Save last N states',
+                value=0,
+                precision=0,
+                info='(Optional) Save only the specified number of states (old models will be deleted)',
+            )
+        with gr.Row():
+            self.keep_tokens = gr.Slider(
+                label='Keep n tokens', value='0', minimum=0, maximum=32, step=1
+            )
+            self.clip_skip = gr.Slider(
+                label='Clip skip', value='1', minimum=1, maximum=12, step=1
+            )
+            self.max_token_length = gr.Dropdown(
+                label='Max Token Length',
+                choices=[
+                    '75',
+                    '150',
+                    '225',
+                ],
+                value='75',
+            )
+            self.full_fp16 = gr.Checkbox(
+                label='Full fp16 training (experimental)', value=False
+            )
+        with gr.Row():
+            self.gradient_checkpointing = gr.Checkbox(
+                label='Gradient checkpointing', value=False
+            )
+            self.shuffle_caption = gr.Checkbox(label='Shuffle caption', value=False)
+            self.persistent_data_loader_workers = gr.Checkbox(
+                label='Persistent data loader', value=False
+            )
+            self.mem_eff_attn = gr.Checkbox(
+                label='Memory efficient attention', value=False
+            )
+        with gr.Row():
+            # This use_8bit_adam element should be removed in a future release as it is no longer used
+            # use_8bit_adam = gr.Checkbox(
+            #     label='Use 8bit adam', value=False, visible=False
+            # )
+            self.xformers = gr.Checkbox(label='Use xformers', value=True)
+            self.color_aug = gr.Checkbox(label='Color augmentation', value=False)
+            self.flip_aug = gr.Checkbox(label='Flip augmentation', value=False)
+            self.min_snr_gamma = gr.Slider(
+                label='Min SNR gamma', value=0, minimum=0, maximum=20, step=1
+            )
+        with gr.Row():
+            self.bucket_no_upscale = gr.Checkbox(
+                label="Don't upscale bucket resolution", value=True
+            )
+            self.bucket_reso_steps = gr.Slider(
+                label='Bucket resolution steps', value=64, minimum=1, maximum=128
+            )
+            self.random_crop = gr.Checkbox(
+                label='Random crop instead of center crop', value=False
+            )
+        
+        with gr.Row():
+            self.min_timestep = gr.Slider(
+                label='Min Timestep',
+                value=0,
+                step=1,
+                minimum=0,
+                maximum=1000,
+                info='Values greater than 0 will make the model more img2img focussed. 0 = image only'
+            )
+            self.max_timestep = gr.Slider(
+                label='Max Timestep',
+                value=1000,
+                step=1,
+                minimum=0,
+                maximum=1000,
+                info='Values lower than 1000 will make the model more img2img focussed. 1000 = noise only',
+            )
+        
+        with gr.Row():
+            self.noise_offset_type = gr.Dropdown(
+                label='Noise offset type',
+                choices=[
+                    'Original',
+                    'Multires',
+                ],
+                value='Original',
+            )
+            with gr.Row(visible=True) as self.noise_offset_original:
+                self.noise_offset = gr.Slider(
+                    label='Noise offset',
+                    value=0,
+                    minimum=0,
+                    maximum=1,
+                    step=0.01,
+                    info='recommended values are 0.05 - 0.15',
+                )
+                self.adaptive_noise_scale = gr.Slider(
+                    label='Adaptive noise scale',
+                    value=0,
+                    minimum=-1,
+                    maximum=1,
+                    step=0.001,
+                    info='(Experimental, Optional) Since the latent is close to a normal distribution, it may be a good idea to specify a value around 1/10 the noise offset.',
+                )
+            with gr.Row(visible=False) as self.noise_offset_multires:
+                self.multires_noise_iterations = gr.Slider(
+                    label='Multires noise iterations',
+                    value=0,
+                    minimum=0,
+                    maximum=64,
+                    step=1,
+                    info='enable multires noise (recommended values are 6-10)',
+                )
+                self.multires_noise_discount = gr.Slider(
+                    label='Multires noise discount',
+                    value=0,
+                    minimum=0,
+                    maximum=1,
+                    step=0.01,
+                    info='recommended values are 0.8. For LoRAs with small datasets, 0.1-0.3',
+                )
+            self.noise_offset_type.change(
+                noise_offset_type_change,
+                inputs=[self.noise_offset_type],
+                outputs=[self.noise_offset_original, self.noise_offset_multires]
+            )
+        with gr.Row():
+            self.caption_dropout_every_n_epochs = gr.Number(
+                label='Dropout caption every n epochs', value=0
+            )
+            self.caption_dropout_rate = gr.Slider(
+                label='Rate of caption dropout', value=0, minimum=0, maximum=1
+            )
+            self.vae_batch_size = gr.Slider(
+                label='VAE batch size', minimum=0, maximum=32, value=0, step=1
+            )
+        with gr.Row():
+            self.save_state = gr.Checkbox(label='Save training state', value=False)
+            self.resume = gr.Textbox(
+                label='Resume from saved training state',
+                placeholder='path to "last-state" state folder to resume from',
+            )
+            self.resume_button = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            self.resume_button.click(
+                get_folder_path,
+                outputs=self.resume,
+                show_progress=False,
+            )
+            self.max_train_epochs = gr.Textbox(
+                label='Max train epoch',
+                placeholder='(Optional) Override number of epoch',
+            )
+            self.max_data_loader_n_workers = gr.Textbox(
+                label='Max num workers for DataLoader',
+                placeholder='(Optional) Override number of epoch. Default: 8',
+                value='0',
+            )
+        with gr.Row():
+            self.wandb_api_key = gr.Textbox(
+                label='WANDB API Key',
+                value='',
+                placeholder='(Optional)',
+                info='Users can obtain and/or generate an api key in the their user settings on the website: https://wandb.ai/login',
+            )
+            self.use_wandb = gr.Checkbox(
+                label='WANDB Logging',
+                value=False,
+                info='If unchecked, tensorboard will be used as the default for logging.',
+            )
+            self.scale_v_pred_loss_like_noise_pred = gr.Checkbox(
+                label='Scale v prediction loss',
+                value=False,
+                info='Only for SD v2 models. By scaling the loss according to the time step, the weights of global noise prediction and local noise prediction become the same, and the improvement of details may be expected.',
+            )
+            
\ No newline at end of file
diff --git a/library/class_basic_training.py b/library/class_basic_training.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a2bce79d7ada735218cc5586852e4d1b807a130
--- /dev/null
+++ b/library/class_basic_training.py
@@ -0,0 +1,131 @@
+import gradio as gr
+import os
+
+class BasicTraining:
+    def __init__(
+        self,
+        learning_rate_value='1e-6',
+        lr_scheduler_value='constant',
+        lr_warmup_value='0',
+        finetuning: bool = False,
+    ):
+        self.learning_rate_value = learning_rate_value
+        self.lr_scheduler_value = lr_scheduler_value
+        self.lr_warmup_value = lr_warmup_value
+        self.finetuning = finetuning
+
+        with gr.Row():
+            self.train_batch_size = gr.Slider(
+                minimum=1,
+                maximum=64,
+                label='Train batch size',
+                value=1,
+                step=1,
+            )
+            self.epoch = gr.Number(label='Epoch', value=1, precision=0)
+            self.save_every_n_epochs = gr.Number(
+                label='Save every N epochs', value=1, precision=0
+            )
+            self.caption_extension = gr.Textbox(
+                label='Caption Extension',
+                placeholder='(Optional) Extension for caption files. default: .caption',
+            )
+        with gr.Row():
+            self.mixed_precision = gr.Dropdown(
+                label='Mixed precision',
+                choices=[
+                    'no',
+                    'fp16',
+                    'bf16',
+                ],
+                value='fp16',
+            )
+            self.save_precision = gr.Dropdown(
+                label='Save precision',
+                choices=[
+                    'float',
+                    'fp16',
+                    'bf16',
+                ],
+                value='fp16',
+            )
+            self.num_cpu_threads_per_process = gr.Slider(
+                minimum=1,
+                maximum=os.cpu_count(),
+                step=1,
+                label='Number of CPU threads per core',
+                value=2,
+            )
+            self.seed = gr.Textbox(
+                label='Seed', placeholder='(Optional) eg:1234'
+            )
+            self.cache_latents = gr.Checkbox(label='Cache latents', value=True)
+            self.cache_latents_to_disk = gr.Checkbox(
+                label='Cache latents to disk', value=False
+            )
+        with gr.Row():
+            self.learning_rate = gr.Number(
+                label='Learning rate', value=learning_rate_value
+            )
+            self.lr_scheduler = gr.Dropdown(
+                label='LR Scheduler',
+                choices=[
+                    'adafactor',
+                    'constant',
+                    'constant_with_warmup',
+                    'cosine',
+                    'cosine_with_restarts',
+                    'linear',
+                    'polynomial',
+                ],
+                value=lr_scheduler_value,
+            )
+            self.lr_warmup = gr.Slider(
+                label='LR warmup (% of steps)',
+                value=lr_warmup_value,
+                minimum=0,
+                maximum=100,
+                step=1,
+            )
+            self.optimizer = gr.Dropdown(
+                label='Optimizer',
+                choices=[
+                    'AdamW',
+                    'AdamW8bit',
+                    'Adafactor',
+                    'DAdaptation',
+                    'DAdaptAdaGrad',
+                    'DAdaptAdam',
+                    'DAdaptAdan',
+                    'DAdaptAdanIP',
+                    'DAdaptAdamPreprint',
+                    'DAdaptLion',
+                    'DAdaptSGD',
+                    'Lion',
+                    'Lion8bit',
+                    'Prodigy',
+                    'SGDNesterov',
+                    'SGDNesterov8bit',
+                ],
+                value='AdamW8bit',
+                interactive=True,
+            )
+        with gr.Row():
+            self.optimizer_args = gr.Textbox(
+                label='Optimizer extra arguments',
+                placeholder='(Optional) eg: relative_step=True scale_parameter=True warmup_init=True',
+            )
+        with gr.Row(visible=not finetuning):
+            self.max_resolution = gr.Textbox(
+                label='Max resolution',
+                value='512,512',
+                placeholder='512,512',
+            )
+            self.stop_text_encoder_training = gr.Slider(
+                minimum=-1,
+                maximum=100,
+                value=0,
+                step=1,
+                label='Stop text encoder training',
+            )
+            self.enable_bucket = gr.Checkbox(label='Enable buckets', value=True)
\ No newline at end of file
diff --git a/library/class_configuration_file.py b/library/class_configuration_file.py
new file mode 100644
index 0000000000000000000000000000000000000000..2eb34bdb096c17483ed851f0c0e47ab7fd8e26c9
--- /dev/null
+++ b/library/class_configuration_file.py
@@ -0,0 +1,30 @@
+import gradio as gr
+from .common_gui import remove_doublequote
+
+
+class ConfigurationFile:
+    def __init__(self, headless=False):
+        self.headless = headless
+        with gr.Accordion('Configuration file', open=False):
+            with gr.Row():
+                self.button_open_config = gr.Button(
+                    'Open 📂', elem_id='open_folder', visible=(not self.headless)
+                )
+                self.button_save_config = gr.Button(
+                    'Save 💾', elem_id='open_folder',
+                )
+                self.button_save_as_config = gr.Button(
+                    'Save as... 💾', elem_id='open_folder', visible=(not self.headless)
+                )
+                self.config_file_name = gr.Textbox(
+                    label='',
+                    placeholder="type the configuration file path or use the 'Open' button above to select it...",
+                    interactive=True,
+                )
+                self.button_load_config = gr.Button('Load 💾', elem_id='open_folder')
+                self.config_file_name.blur(
+                    remove_doublequote,
+                    inputs=[self.config_file_name],
+                    outputs=[self.config_file_name],
+                )
+                
diff --git a/library/class_dreambooth_gui.py b/library/class_dreambooth_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a4473b83705d5daffe63ab4b732221cffb5c0e8
--- /dev/null
+++ b/library/class_dreambooth_gui.py
@@ -0,0 +1,104 @@
+import gradio as gr
+import json
+from .class_configuration_file import ConfigurationFile
+from .class_source_model import SourceModel
+from .class_folders import Folders
+from .class_basic_training import BasicTraining
+from .class_advanced_training import AdvancedTraining
+from .class_sample_images import SampleImages
+from library.dreambooth_folder_creation_gui import (
+    gradio_dreambooth_folder_creation_tab,
+)
+from .common_gui import color_aug_changed
+
+class Dreambooth:
+    def __init__(
+        self,
+        headless: bool = False,
+    ):
+        self.headless = headless
+        self.dummy_db_true = gr.Label(value=True, visible=False)
+        self.dummy_db_false = gr.Label(value=False, visible=False)
+        self.dummy_headless = gr.Label(value=headless, visible=False)
+
+        gr.Markdown('Train a custom model using kohya dreambooth python code...')
+    
+        # Setup Configuration Files Gradio
+        self.config = ConfigurationFile(headless)
+        
+        self.source_model = SourceModel(headless=headless)
+        
+        with gr.Tab('Folders'):
+            self.folders = Folders(headless=headless)
+        with gr.Tab('Parameters'):
+            self.basic_training = BasicTraining(
+                learning_rate_value='1e-5',
+                lr_scheduler_value='cosine',
+                lr_warmup_value='10',
+            )
+            with gr.Accordion('Advanced Configuration', open=False):
+                self.advanced_training = AdvancedTraining(headless=headless)
+                self.advanced_training.color_aug.change(
+                    color_aug_changed,
+                    inputs=[self.advanced_training.color_aug],
+                    outputs=[self.basic_training.cache_latents],
+                )
+
+            self.sample = SampleImages()
+
+        with gr.Tab('Tools'):
+            gr.Markdown(
+                'This section provide Dreambooth tools to help setup your dataset...'
+            )
+            gradio_dreambooth_folder_creation_tab(
+                train_data_dir_input=self.folders.train_data_dir,
+                reg_data_dir_input=self.folders.reg_data_dir,
+                output_dir_input=self.folders.output_dir,
+                logging_dir_input=self.folders.logging_dir,
+                headless=headless,
+            )
+            
+    def save_to_json(self, filepath):
+        def serialize(obj):
+            if isinstance(obj, gr.inputs.Input):
+                return obj.get()
+            if isinstance(obj, (bool, int, float, str)):
+                return obj
+            if isinstance(obj, dict):
+                return {k: serialize(v) for k, v in obj.items()}
+            if hasattr(obj, "__dict__"):
+                return serialize(vars(obj))
+            return str(obj)  # Fallback for objects that can't be serialized
+
+        try:
+            with open(filepath, 'w') as outfile:
+                print(serialize(vars(self)))
+                json.dump(serialize(vars(self)), outfile)
+        except Exception as e:
+            print(f"Error saving to JSON: {str(e)}")
+
+    def load_from_json(self, filepath):
+        def deserialize(key, value):
+            if hasattr(self, key):
+                attr = getattr(self, key)
+                if isinstance(attr, gr.inputs.Input):
+                    attr.set(value)
+                elif hasattr(attr, "__dict__"):
+                    for k, v in value.items():
+                        deserialize(k, v)
+                else:
+                    setattr(self, key, value)
+            else:
+                print(f"Warning: {key} not found in the object's attributes.")
+
+        try:
+            with open(filepath) as json_file:
+                data = json.load(json_file)
+                for key, value in data.items():
+                    deserialize(key, value)
+        except FileNotFoundError:
+            print(f"Error: The file {filepath} was not found.")
+        except json.JSONDecodeError:
+            print(f"Error: The file {filepath} could not be decoded as JSON.")
+        except Exception as e:
+            print(f"Error loading from JSON: {str(e)}")
\ No newline at end of file
diff --git a/library/class_folders.py b/library/class_folders.py
new file mode 100644
index 0000000000000000000000000000000000000000..486f07fa8896b7aa4eae8f5c90e9cdca9f41b4a4
--- /dev/null
+++ b/library/class_folders.py
@@ -0,0 +1,89 @@
+import gradio as gr
+from .common_gui import remove_doublequote, get_folder_path
+
+class Folders:
+    def __init__(self, headless=False):
+        self.headless = headless
+        
+        with gr.Row():
+            self.train_data_dir = gr.Textbox(
+                label='Image folder',
+                placeholder='Folder where the training folders containing the images are located',
+            )
+            self.train_data_dir_folder = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not self.headless)
+            )
+            self.train_data_dir_folder.click(
+                get_folder_path,
+                outputs=self.train_data_dir,
+                show_progress=False,
+            )
+            self.reg_data_dir = gr.Textbox(
+                label='Regularisation folder',
+                placeholder='(Optional) Folder where where the regularization folders containing the images are located',
+            )
+            self.reg_data_dir_folder = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not self.headless)
+            )
+            self.reg_data_dir_folder.click(
+                get_folder_path,
+                outputs=self.reg_data_dir,
+                show_progress=False,
+            )
+        with gr.Row():
+            self.output_dir = gr.Textbox(
+                label='Output folder',
+                placeholder='Folder to output trained model',
+            )
+            self.output_dir_folder = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not self.headless)
+            )
+            self.output_dir_folder.click(
+                get_folder_path,
+                outputs=self.output_dir,
+                show_progress=False,
+            )
+            self.logging_dir = gr.Textbox(
+                label='Logging folder',
+                placeholder='Optional: enable logging and output TensorBoard log to this folder',
+            )
+            self.logging_dir_folder = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not self.headless)
+            )
+            self.logging_dir_folder.click(
+                get_folder_path,
+                outputs=self.logging_dir,
+                show_progress=False,
+            )
+        with gr.Row():
+            self.output_name = gr.Textbox(
+                label='Model output name',
+                placeholder='(Name of the model to output)',
+                value='last',
+                interactive=True,
+            )
+            self.training_comment = gr.Textbox(
+                label='Training comment',
+                placeholder='(Optional) Add training comment to be included in metadata',
+                interactive=True,
+            )
+        self.train_data_dir.blur(
+            remove_doublequote,
+            inputs=[self.train_data_dir],
+            outputs=[self.train_data_dir],
+        )
+        self.reg_data_dir.blur(
+            remove_doublequote,
+            inputs=[self.reg_data_dir],
+            outputs=[self.reg_data_dir],
+        )
+        self.output_dir.blur(
+            remove_doublequote,
+            inputs=[self.output_dir],
+            outputs=[self.output_dir],
+        )
+        self.logging_dir.blur(
+            remove_doublequote,
+            inputs=[self.logging_dir],
+            outputs=[self.logging_dir],
+        )
\ No newline at end of file
diff --git a/library/class_lora_tab.py b/library/class_lora_tab.py
new file mode 100644
index 0000000000000000000000000000000000000000..a19f34a9f54b583e10d421094a392bbda8c2cd9f
--- /dev/null
+++ b/library/class_lora_tab.py
@@ -0,0 +1,42 @@
+import gradio as gr
+from library.merge_lora_gui import gradio_merge_lora_tab
+from library.svd_merge_lora_gui import gradio_svd_merge_lora_tab
+from library.verify_lora_gui import gradio_verify_lora_tab
+from library.resize_lora_gui import gradio_resize_lora_tab
+from library.extract_lora_gui import gradio_extract_lora_tab
+from library.extract_lycoris_locon_gui import gradio_extract_lycoris_locon_tab
+from library.extract_lora_from_dylora_gui import gradio_extract_dylora_tab
+from library.merge_lycoris_gui import gradio_merge_lycoris_tab
+
+# Deprecated code
+from library.dataset_balancing_gui import gradio_dataset_balancing_tab
+from library.dreambooth_folder_creation_gui import (
+    gradio_dreambooth_folder_creation_tab,
+)
+
+class LoRATools:
+    def __init__(self, folders = "", headless:bool = False):
+        self.headless = headless
+        self.folders = folders
+        
+        gr.Markdown(
+            'This section provide LoRA tools to help setup your dataset...'
+        )
+        gradio_extract_dylora_tab(headless=headless)
+        gradio_extract_lora_tab(headless=headless)
+        gradio_extract_lycoris_locon_tab(headless=headless)
+        gradio_merge_lora_tab(headless=headless)
+        gradio_merge_lycoris_tab(headless=headless)
+        gradio_svd_merge_lora_tab(headless=headless)
+        gradio_resize_lora_tab(headless=headless)
+        gradio_verify_lora_tab(headless=headless)
+        if folders:
+            with gr.Tab('Deprecated'):
+                gradio_dreambooth_folder_creation_tab(
+                    train_data_dir_input=folders.train_data_dir,
+                    reg_data_dir_input=folders.reg_data_dir,
+                    output_dir_input=folders.output_dir,
+                    logging_dir_input=folders.logging_dir,
+                    headless=headless,
+                )
+                gradio_dataset_balancing_tab(headless=headless)
\ No newline at end of file
diff --git a/library/class_sample_images.py b/library/class_sample_images.py
new file mode 100644
index 0000000000000000000000000000000000000000..70cc57ebd7cc688f41f7c8260767b9d41916f4b1
--- /dev/null
+++ b/library/class_sample_images.py
@@ -0,0 +1,158 @@
+import tempfile
+import os
+import gradio as gr
+from easygui import msgbox
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+
+
+###
+### Gradio common sampler GUI section
+###
+
+def sample_gradio_config():
+    with gr.Accordion('Sample images config', open=False):
+        with gr.Row():
+            sample_every_n_steps = gr.Number(
+                label='Sample every n steps',
+                value=0,
+                precision=0,
+                interactive=True,
+            )
+            sample_every_n_epochs = gr.Number(
+                label='Sample every n epochs',
+                value=0,
+                precision=0,
+                interactive=True,
+            )
+            sample_sampler = gr.Dropdown(
+                label='Sample sampler',
+                choices=[
+                    'ddim',
+                    'pndm',
+                    'lms',
+                    'euler',
+                    'euler_a',
+                    'heun',
+                    'dpm_2',
+                    'dpm_2_a',
+                    'dpmsolver',
+                    'dpmsolver++',
+                    'dpmsingle',
+                    'k_lms',
+                    'k_euler',
+                    'k_euler_a',
+                    'k_dpm_2',
+                    'k_dpm_2_a',
+                ],
+                value='euler_a',
+                interactive=True,
+            )
+        with gr.Row():
+            sample_prompts = gr.Textbox(
+                lines=5,
+                label='Sample prompts',
+                interactive=True,
+                placeholder='masterpiece, best quality, 1girl, in white shirts, upper body, looking at viewer, simple background --n low quality, worst quality, bad anatomy,bad composition, poor, low effort --w 768 --h 768 --d 1 --l 7.5 --s 28',
+                info='Enter one sample prompt per line to generate multiple samples per cycle. Optional specifiers include: --w (width), --h (height), --d (seed), --l (cfg scale), --s (sampler steps) and --n (negative prompt). To modify sample prompts during training, edit the prompt.txt file in the samples directory.'
+            )
+    return (
+        sample_every_n_steps,
+        sample_every_n_epochs,
+        sample_sampler,
+        sample_prompts,
+    )
+    
+def run_cmd_sample(
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    output_dir,
+):
+    output_dir = os.path.join(output_dir, 'sample')
+
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    run_cmd = ''
+
+    if sample_every_n_epochs == 0 and sample_every_n_steps == 0:
+        return run_cmd
+
+    # Create the prompt file and get its path
+    sample_prompts_path = os.path.join(output_dir, 'prompt.txt')
+
+    with open(sample_prompts_path, 'w') as f:
+        f.write(sample_prompts)
+
+    run_cmd += f' --sample_sampler={sample_sampler}'
+    run_cmd += f' --sample_prompts="{sample_prompts_path}"'
+
+    if not sample_every_n_epochs == 0:
+        run_cmd += f' --sample_every_n_epochs="{sample_every_n_epochs}"'
+
+    if not sample_every_n_steps == 0:
+        run_cmd += f' --sample_every_n_steps="{sample_every_n_steps}"'
+
+    return run_cmd    
+
+
+class SampleImages:
+    def __init__(
+        self,
+    ):
+        with gr.Accordion('Sample images config', open=False):
+            with gr.Row():
+                self.sample_every_n_steps = gr.Number(
+                    label='Sample every n steps',
+                    value=0,
+                    precision=0,
+                    interactive=True,
+                )
+                self.sample_every_n_epochs = gr.Number(
+                    label='Sample every n epochs',
+                    value=0,
+                    precision=0,
+                    interactive=True,
+                )
+                self.sample_sampler = gr.Dropdown(
+                    label='Sample sampler',
+                    choices=[
+                        'ddim',
+                        'pndm',
+                        'lms',
+                        'euler',
+                        'euler_a',
+                        'heun',
+                        'dpm_2',
+                        'dpm_2_a',
+                        'dpmsolver',
+                        'dpmsolver++',
+                        'dpmsingle',
+                        'k_lms',
+                        'k_euler',
+                        'k_euler_a',
+                        'k_dpm_2',
+                        'k_dpm_2_a',
+                    ],
+                    value='euler_a',
+                    interactive=True,
+                )
+            with gr.Row():
+                self.sample_prompts = gr.Textbox(
+                    lines=5,
+                    label='Sample prompts',
+                    interactive=True,
+                    placeholder='masterpiece, best quality, 1girl, in white shirts, upper body, looking at viewer, simple background --n low quality, worst quality, bad anatomy,bad composition, poor, low effort --w 768 --h 768 --d 1 --l 7.5 --s 28',
+                    info='Enter one sample prompt per line to generate multiple samples per cycle. Optional specifiers include: --w (width), --h (height), --d (seed), --l (cfg scale), --s (sampler steps) and --n (negative prompt). To modify sample prompts during training, edit the prompt.txt file in the samples directory.'
+                )
+                
\ No newline at end of file
diff --git a/library/class_sdxl_parameters.py b/library/class_sdxl_parameters.py
new file mode 100644
index 0000000000000000000000000000000000000000..33af8631b0234c9e656975f60ad8dd1d97faedb0
--- /dev/null
+++ b/library/class_sdxl_parameters.py
@@ -0,0 +1,23 @@
+import gradio as gr
+
+### SDXL Parameters class
+class SDXLParameters:
+    def __init__(self, sdxl_checkbox, show_sdxl_cache_text_encoder_outputs:bool = True):
+        self.sdxl_checkbox = sdxl_checkbox
+        self.show_sdxl_cache_text_encoder_outputs = show_sdxl_cache_text_encoder_outputs
+
+        with gr.Accordion(visible=False, open=True, label='SDXL Specific Parameters') as self.sdxl_row:
+            with gr.Row():
+                self.sdxl_cache_text_encoder_outputs = gr.Checkbox(
+                    label='Cache text encoder outputs',
+                    info='Cache the outputs of the text encoders. This option is useful to reduce the GPU memory usage. This option cannot be used with options for shuffling or dropping the captions.',
+                    value=False,
+                    visible=show_sdxl_cache_text_encoder_outputs
+                )
+                self.sdxl_no_half_vae = gr.Checkbox(
+                    label='No half VAE',
+                    info='Disable the half-precision (mixed-precision) VAE. VAE for SDXL seems to produce NaNs in some cases. This option is useful to avoid the NaNs.',
+                    value=True
+                )
+
+        self.sdxl_checkbox.change(lambda sdxl_checkbox: gr.Accordion.update(visible=sdxl_checkbox), inputs=[self.sdxl_checkbox], outputs=[self.sdxl_row])
diff --git a/library/class_source_model.py b/library/class_source_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..509bc41e5d14888456bc4dd9a3139def5b124361
--- /dev/null
+++ b/library/class_source_model.py
@@ -0,0 +1,112 @@
+import gradio as gr
+from .common_gui import (
+    get_any_file_path,
+    get_folder_path,
+    set_pretrained_model_name_or_path_input,
+)
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+
+
+class SourceModel:
+    def __init__(
+        self,
+        save_model_as_choices=[
+            'same as source model',
+            'ckpt',
+            'diffusers',
+            'diffusers_safetensors',
+            'safetensors',
+        ],
+        headless=False,
+    ):
+        self.headless = headless
+        self.save_model_as_choices = save_model_as_choices
+
+        with gr.Tab('Source model'):
+            # Define the input elements
+            with gr.Row():
+                self.model_list = gr.Dropdown(
+                    label='Model Quick Pick',
+                    choices=[
+                        'custom',
+                        # 'stabilityai/stable-diffusion-xl-base-0.9',
+                        # 'stabilityai/stable-diffusion-xl-refiner-0.9',
+                        'stabilityai/stable-diffusion-2-1-base/blob/main/v2-1_512-ema-pruned',
+                        'stabilityai/stable-diffusion-2-1-base',
+                        'stabilityai/stable-diffusion-2-base',
+                        'stabilityai/stable-diffusion-2-1/blob/main/v2-1_768-ema-pruned',
+                        'stabilityai/stable-diffusion-2-1',
+                        'stabilityai/stable-diffusion-2',
+                        'runwayml/stable-diffusion-v1-5',
+                        'CompVis/stable-diffusion-v1-4',
+                    ],
+                    value='runwayml/stable-diffusion-v1-5',
+                )
+                self.save_model_as = gr.Dropdown(
+                    label='Save trained model as',
+                    choices=save_model_as_choices,
+                    value='safetensors',
+                )
+            with gr.Row():
+                self.pretrained_model_name_or_path = gr.Textbox(
+                    label='Pretrained model name or path',
+                    placeholder='enter the path to custom model or name of pretrained model',
+                    value='runwayml/stable-diffusion-v1-5',
+                    visible=(False and not headless),
+                )
+                self.pretrained_model_name_or_path_file = gr.Button(
+                    document_symbol,
+                    elem_id='open_folder_small',
+                    visible=(False and not headless),
+                )
+                self.pretrained_model_name_or_path_file.click(
+                    get_any_file_path,
+                    inputs=self.pretrained_model_name_or_path,
+                    outputs=self.pretrained_model_name_or_path,
+                    show_progress=False,
+                )
+                self.pretrained_model_name_or_path_folder = gr.Button(
+                    folder_symbol,
+                    elem_id='open_folder_small',
+                    visible=(False and not headless),
+                )
+                self.pretrained_model_name_or_path_folder.click(
+                    get_folder_path,
+                    inputs=self.pretrained_model_name_or_path,
+                    outputs=self.pretrained_model_name_or_path,
+                    show_progress=False,
+                )
+                self.v2 = gr.Checkbox(label='v2', value=False, visible=False)
+                self.v_parameterization = gr.Checkbox(
+                    label='v_parameterization', value=False, visible=False
+                )
+                self.sdxl_checkbox = gr.Checkbox(
+                    label='SDXL Model', value=False, visible=False
+                )
+
+            self.model_list.change(
+                set_pretrained_model_name_or_path_input,
+                inputs=[
+                    self.model_list,
+                    self.pretrained_model_name_or_path,
+                    self.pretrained_model_name_or_path_file,
+                    self.pretrained_model_name_or_path_folder,
+                    self.v2,
+                    self.v_parameterization,
+                    self.sdxl_checkbox,
+                ],
+                outputs=[
+                    self.model_list,
+                    self.pretrained_model_name_or_path,
+                    self.pretrained_model_name_or_path_file,
+                    self.pretrained_model_name_or_path_folder,
+                    self.v2,
+                    self.v_parameterization,
+                    self.sdxl_checkbox,
+                ],
+                show_progress=False,
+            )
diff --git a/library/common_gui.py b/library/common_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..a71213caa3c208df8616a9b7510ed4ba6d3700d9
--- /dev/null
+++ b/library/common_gui.py
@@ -0,0 +1,842 @@
+from tkinter import filedialog, Tk
+from easygui import msgbox
+import os
+import re
+import gradio as gr
+import easygui
+import shutil
+import sys
+import json
+
+from library.custom_logging import setup_logging
+from datetime import datetime
+
+# Set up logging
+log = setup_logging()
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+
+# define a list of substrings to search for v2 base models
+V2_BASE_MODELS = [
+    'stabilityai/stable-diffusion-2-1-base/blob/main/v2-1_512-ema-pruned',
+    'stabilityai/stable-diffusion-2-1-base',
+    'stabilityai/stable-diffusion-2-base',
+]
+
+# define a list of substrings to search for v_parameterization models
+V_PARAMETERIZATION_MODELS = [
+    'stabilityai/stable-diffusion-2-1/blob/main/v2-1_768-ema-pruned',
+    'stabilityai/stable-diffusion-2-1',
+    'stabilityai/stable-diffusion-2',
+]
+
+# define a list of substrings to v1.x models
+V1_MODELS = [
+    'CompVis/stable-diffusion-v1-4',
+    'runwayml/stable-diffusion-v1-5',
+]
+
+# define a list of substrings to search for SDXL base models
+SDXL_MODELS = [
+    'stabilityai/stable-diffusion-xl-base-0.9',
+    'stabilityai/stable-diffusion-xl-refiner-0.9'
+]
+
+# define a list of substrings to search for
+ALL_PRESET_MODELS = V2_BASE_MODELS + V_PARAMETERIZATION_MODELS + V1_MODELS + SDXL_MODELS
+
+ENV_EXCLUSION = ['COLAB_GPU', 'RUNPOD_POD_ID']
+
+
+def check_if_model_exist(
+    output_name, output_dir, save_model_as, headless=False
+):
+    if headless:
+        log.info(
+            'Headless mode, skipping verification if model already exist... if model already exist it will be overwritten...'
+        )
+        return False
+
+    if save_model_as in ['diffusers', 'diffusers_safetendors']:
+        ckpt_folder = os.path.join(output_dir, output_name)
+        if os.path.isdir(ckpt_folder):
+            msg = f'A diffuser model with the same name {ckpt_folder} already exists. Do you want to overwrite it?'
+            if not easygui.ynbox(msg, 'Overwrite Existing Model?'):
+                log.info(
+                    'Aborting training due to existing model with same name...'
+                )
+                return True
+    elif save_model_as in ['ckpt', 'safetensors']:
+        ckpt_file = os.path.join(output_dir, output_name + '.' + save_model_as)
+        if os.path.isfile(ckpt_file):
+            msg = f'A model with the same file name {ckpt_file} already exists. Do you want to overwrite it?'
+            if not easygui.ynbox(msg, 'Overwrite Existing Model?'):
+                log.info(
+                    'Aborting training due to existing model with same name...'
+                )
+                return True
+    else:
+        log.info(
+            'Can\'t verify if existing model exist when save model is set a "same as source model", continuing to train model...'
+        )
+        return False
+
+    return False
+
+
+def output_message(msg='', title='', headless=False):
+    if headless:
+        log.info(msg)
+    else:
+        msgbox(msg=msg, title=title)
+
+
+def update_my_data(my_data):
+    # Update the optimizer based on the use_8bit_adam flag
+    use_8bit_adam = my_data.get('use_8bit_adam', False)
+    my_data.setdefault('optimizer', 'AdamW8bit' if use_8bit_adam else 'AdamW')
+
+    # Update model_list to custom if empty or pretrained_model_name_or_path is not a preset model
+    model_list = my_data.get('model_list', [])
+    pretrained_model_name_or_path = my_data.get(
+        'pretrained_model_name_or_path', ''
+    )
+    if (
+        not model_list
+        or pretrained_model_name_or_path not in ALL_PRESET_MODELS
+    ):
+        my_data['model_list'] = 'custom'
+
+    # Convert values to int if they are strings
+    for key in ['epoch', 'save_every_n_epochs', 'lr_warmup']:
+        value = my_data.get(key, 0)
+        if isinstance(value, str) and value.strip().isdigit():
+            my_data[key] = int(value)
+        elif not value:
+            my_data[key] = 0
+
+    # Convert values to float if they are strings
+    for key in ['noise_offset', 'learning_rate', 'text_encoder_lr', 'unet_lr']:
+        value = my_data.get(key, 0)
+        if isinstance(value, str) and value.strip().isdigit():
+            my_data[key] = float(value)
+        elif not value:
+            my_data[key] = 0
+
+    # Update LoRA_type if it is set to LoCon
+    if my_data.get('LoRA_type', 'Standard') == 'LoCon':
+        my_data['LoRA_type'] = 'LyCORIS/LoCon'
+
+    # Update model save choices due to changes for LoRA and TI training
+    if 'save_model_as' in my_data:
+        if (
+            my_data.get('LoRA_type') or my_data.get('num_vectors_per_token')
+        ) and my_data.get('save_model_as') not in ['safetensors', 'ckpt']:
+            message = 'Updating save_model_as to safetensors because the current value in the config file is no longer applicable to {}'
+            if my_data.get('LoRA_type'):
+                log.info(message.format('LoRA'))
+            if my_data.get('num_vectors_per_token'):
+                log.info(message.format('TI'))
+            my_data['save_model_as'] = 'safetensors'
+
+    return my_data
+
+
+def get_dir_and_file(file_path):
+    dir_path, file_name = os.path.split(file_path)
+    return (dir_path, file_name)
+
+
+def get_file_path(
+    file_path='', default_extension='.json', extension_name='Config files'
+):
+    if (
+        not any(var in os.environ for var in ENV_EXCLUSION)
+        and sys.platform != 'darwin'
+    ):
+        current_file_path = file_path
+        # log.info(f'current file path: {current_file_path}')
+
+        initial_dir, initial_file = get_dir_and_file(file_path)
+
+        # Create a hidden Tkinter root window
+        root = Tk()
+        root.wm_attributes('-topmost', 1)
+        root.withdraw()
+
+        # Show the open file dialog and get the selected file path
+        file_path = filedialog.askopenfilename(
+            filetypes=(
+                (extension_name, f'*{default_extension}'),
+                ('All files', '*.*'),
+            ),
+            defaultextension=default_extension,
+            initialfile=initial_file,
+            initialdir=initial_dir,
+        )
+
+        # Destroy the hidden root window
+        root.destroy()
+
+        # If no file is selected, use the current file path
+        if not file_path:
+            file_path = current_file_path
+        current_file_path = file_path
+        # log.info(f'current file path: {current_file_path}')
+
+    return file_path
+
+
+def get_any_file_path(file_path=''):
+    if (
+        not any(var in os.environ for var in ENV_EXCLUSION)
+        and sys.platform != 'darwin'
+    ):
+        current_file_path = file_path
+        # log.info(f'current file path: {current_file_path}')
+
+        initial_dir, initial_file = get_dir_and_file(file_path)
+
+        root = Tk()
+        root.wm_attributes('-topmost', 1)
+        root.withdraw()
+        file_path = filedialog.askopenfilename(
+            initialdir=initial_dir,
+            initialfile=initial_file,
+        )
+        root.destroy()
+
+        if file_path == '':
+            file_path = current_file_path
+
+    return file_path
+
+
+def remove_doublequote(file_path):
+    if file_path != None:
+        file_path = file_path.replace('"', '')
+
+    return file_path
+
+
+def get_folder_path(folder_path=''):
+    if (
+        not any(var in os.environ for var in ENV_EXCLUSION)
+        and sys.platform != 'darwin'
+    ):
+        current_folder_path = folder_path
+
+        initial_dir, initial_file = get_dir_and_file(folder_path)
+
+        root = Tk()
+        root.wm_attributes('-topmost', 1)
+        root.withdraw()
+        folder_path = filedialog.askdirectory(initialdir=initial_dir)
+        root.destroy()
+
+        if folder_path == '':
+            folder_path = current_folder_path
+
+    return folder_path
+
+
+def get_saveasfile_path(
+    file_path='', defaultextension='.json', extension_name='Config files'
+):
+    if (
+        not any(var in os.environ for var in ENV_EXCLUSION)
+        and sys.platform != 'darwin'
+    ):
+        current_file_path = file_path
+        # log.info(f'current file path: {current_file_path}')
+
+        initial_dir, initial_file = get_dir_and_file(file_path)
+
+        root = Tk()
+        root.wm_attributes('-topmost', 1)
+        root.withdraw()
+        save_file_path = filedialog.asksaveasfile(
+            filetypes=(
+                (f'{extension_name}', f'{defaultextension}'),
+                ('All files', '*'),
+            ),
+            defaultextension=defaultextension,
+            initialdir=initial_dir,
+            initialfile=initial_file,
+        )
+        root.destroy()
+
+        # log.info(save_file_path)
+
+        if save_file_path == None:
+            file_path = current_file_path
+        else:
+            log.info(save_file_path.name)
+            file_path = save_file_path.name
+
+        # log.info(file_path)
+
+    return file_path
+
+
+def get_saveasfilename_path(
+    file_path='', extensions='*', extension_name='Config files'
+):
+    if (
+        not any(var in os.environ for var in ENV_EXCLUSION)
+        and sys.platform != 'darwin'
+    ):
+        current_file_path = file_path
+        # log.info(f'current file path: {current_file_path}')
+
+        initial_dir, initial_file = get_dir_and_file(file_path)
+
+        root = Tk()
+        root.wm_attributes('-topmost', 1)
+        root.withdraw()
+        save_file_path = filedialog.asksaveasfilename(
+            filetypes=(
+                (f'{extension_name}', f'{extensions}'),
+                ('All files', '*'),
+            ),
+            defaultextension=extensions,
+            initialdir=initial_dir,
+            initialfile=initial_file,
+        )
+        root.destroy()
+
+        if save_file_path == '':
+            file_path = current_file_path
+        else:
+            # log.info(save_file_path)
+            file_path = save_file_path
+
+    return file_path
+
+
+def add_pre_postfix(
+    folder: str = '',
+    prefix: str = '',
+    postfix: str = '',
+    caption_file_ext: str = '.caption',
+) -> None:
+    """
+    Add prefix and/or postfix to the content of caption files within a folder.
+    If no caption files are found, create one with the requested prefix and/or postfix.
+
+    Args:
+        folder (str): Path to the folder containing caption files.
+        prefix (str, optional): Prefix to add to the content of the caption files.
+        postfix (str, optional): Postfix to add to the content of the caption files.
+        caption_file_ext (str, optional): Extension of the caption files.
+    """
+
+    if prefix == '' and postfix == '':
+        return
+
+    image_extensions = ('.jpg', '.jpeg', '.png', '.webp')
+    image_files = [
+        f for f in os.listdir(folder) if f.lower().endswith(image_extensions)
+    ]
+
+    for image_file in image_files:
+        caption_file_name = os.path.splitext(image_file)[0] + caption_file_ext
+        caption_file_path = os.path.join(folder, caption_file_name)
+
+        if not os.path.exists(caption_file_path):
+            with open(caption_file_path, 'w', encoding='utf8') as f:
+                separator = ' ' if prefix and postfix else ''
+                f.write(f'{prefix}{separator}{postfix}')
+        else:
+            with open(caption_file_path, 'r+', encoding='utf8') as f:
+                content = f.read()
+                content = content.rstrip()
+                f.seek(0, 0)
+
+                prefix_separator = ' ' if prefix else ''
+                postfix_separator = ' ' if postfix else ''
+                f.write(
+                    f'{prefix}{prefix_separator}{content}{postfix_separator}{postfix}'
+                )
+
+
+def has_ext_files(folder_path: str, file_extension: str) -> bool:
+    """
+    Check if there are any files with the specified extension in the given folder.
+
+    Args:
+        folder_path (str): Path to the folder containing files.
+        file_extension (str): Extension of the files to look for.
+
+    Returns:
+        bool: True if files with the specified extension are found, False otherwise.
+    """
+    for file in os.listdir(folder_path):
+        if file.endswith(file_extension):
+            return True
+    return False
+
+
+def find_replace(
+    folder_path: str = '',
+    caption_file_ext: str = '.caption',
+    search_text: str = '',
+    replace_text: str = '',
+) -> None:
+    """
+    Find and replace text in caption files within a folder.
+
+    Args:
+        folder_path (str, optional): Path to the folder containing caption files.
+        caption_file_ext (str, optional): Extension of the caption files.
+        search_text (str, optional): Text to search for in the caption files.
+        replace_text (str, optional): Text to replace the search text with.
+    """
+    log.info('Running caption find/replace')
+
+    if not has_ext_files(folder_path, caption_file_ext):
+        msgbox(
+            f'No files with extension {caption_file_ext} were found in {folder_path}...'
+        )
+        return
+
+    if search_text == '':
+        return
+
+    caption_files = [
+        f for f in os.listdir(folder_path) if f.endswith(caption_file_ext)
+    ]
+
+    for caption_file in caption_files:
+        with open(
+            os.path.join(folder_path, caption_file), 'r', errors='ignore'
+        ) as f:
+            content = f.read()
+
+        content = content.replace(search_text, replace_text)
+
+        with open(os.path.join(folder_path, caption_file), 'w') as f:
+            f.write(content)
+
+
+def color_aug_changed(color_aug):
+    if color_aug:
+        msgbox(
+            'Disabling "Cache latent" because "Color augmentation" has been selected...'
+        )
+        return gr.Checkbox.update(value=False, interactive=False)
+    else:
+        return gr.Checkbox.update(value=True, interactive=True)
+
+
+def save_inference_file(output_dir, v2, v_parameterization, output_name):
+    # List all files in the directory
+    files = os.listdir(output_dir)
+
+    # Iterate over the list of files
+    for file in files:
+        # Check if the file starts with the value of output_name
+        if file.startswith(output_name):
+            # Check if it is a file or a directory
+            if os.path.isfile(os.path.join(output_dir, file)):
+                # Split the file name and extension
+                file_name, ext = os.path.splitext(file)
+
+                # Copy the v2-inference-v.yaml file to the current file, with a .yaml extension
+                if v2 and v_parameterization:
+                    log.info(
+                        f'Saving v2-inference-v.yaml as {output_dir}/{file_name}.yaml'
+                    )
+                    shutil.copy(
+                        f'./v2_inference/v2-inference-v.yaml',
+                        f'{output_dir}/{file_name}.yaml',
+                    )
+                elif v2:
+                    log.info(
+                        f'Saving v2-inference.yaml as {output_dir}/{file_name}.yaml'
+                    )
+                    shutil.copy(
+                        f'./v2_inference/v2-inference.yaml',
+                        f'{output_dir}/{file_name}.yaml',
+                    )
+
+
+def set_pretrained_model_name_or_path_input(
+    model_list, pretrained_model_name_or_path, pretrained_model_name_or_path_file, pretrained_model_name_or_path_folder, v2, v_parameterization, sdxl
+):
+    # Check if the given model_list is in the list of SDXL models
+    if str(model_list) in SDXL_MODELS:
+        log.info('SDXL model selected. Setting sdxl parameters')
+        v2 = gr.Checkbox.update(value=False, visible=False)
+        v_parameterization = gr.Checkbox.update(value=False, visible=False)
+        sdxl = gr.Checkbox.update(value=True, visible=False)
+        pretrained_model_name_or_path = gr.Textbox.update(value=str(model_list), visible=False)
+        pretrained_model_name_or_path_file = gr.Button.update(visible=False)
+        pretrained_model_name_or_path_folder = gr.Button.update(visible=False)
+        return model_list, pretrained_model_name_or_path, pretrained_model_name_or_path_file, pretrained_model_name_or_path_folder, v2, v_parameterization, sdxl
+
+    # Check if the given model_list is in the list of V2 base models
+    if str(model_list) in V2_BASE_MODELS:
+        log.info('SD v2 base model selected. Setting --v2 parameter')
+        v2 = gr.Checkbox.update(value=True, visible=False)
+        v_parameterization = gr.Checkbox.update(value=False, visible=False)
+        sdxl = gr.Checkbox.update(value=False, visible=False)
+        pretrained_model_name_or_path = gr.Textbox.update(value=str(model_list), visible=False)
+        pretrained_model_name_or_path_file = gr.Button.update(visible=False)
+        pretrained_model_name_or_path_folder = gr.Button.update(visible=False)
+        return model_list, pretrained_model_name_or_path, pretrained_model_name_or_path_file, pretrained_model_name_or_path_folder, v2, v_parameterization, sdxl
+
+    # Check if the given model_list is in the list of V parameterization models
+    if str(model_list) in V_PARAMETERIZATION_MODELS:
+        log.info(
+            'SD v2 model selected. Setting --v2 and --v_parameterization parameters'
+        )
+        v2 = gr.Checkbox.update(value=True, visible=False)
+        v_parameterization = gr.Checkbox.update(value=True, visible=False)
+        sdxl = gr.Checkbox.update(value=False, visible=False)
+        pretrained_model_name_or_path = gr.Textbox.update(value=str(model_list), visible=False)
+        pretrained_model_name_or_path_file = gr.Button.update(visible=False)
+        pretrained_model_name_or_path_folder = gr.Button.update(visible=False)
+        return model_list, pretrained_model_name_or_path, pretrained_model_name_or_path_file, pretrained_model_name_or_path_folder, v2, v_parameterization, sdxl
+
+    # Check if the given model_list is in the list of V1 models
+    if str(model_list) in V1_MODELS:
+        log.info(
+            'SD v1.4 model selected.'
+        )
+        v2 = gr.Checkbox.update(value=False, visible=False)
+        v_parameterization = gr.Checkbox.update(value=False, visible=False)
+        sdxl = gr.Checkbox.update(value=False, visible=False)
+        pretrained_model_name_or_path = gr.Textbox.update(value=str(model_list), visible=False)
+        pretrained_model_name_or_path_file = gr.Button.update(visible=False)
+        pretrained_model_name_or_path_folder = gr.Button.update(visible=False)
+        return model_list, pretrained_model_name_or_path, pretrained_model_name_or_path_file, pretrained_model_name_or_path_folder, v2, v_parameterization, sdxl
+
+    # Check if the model_list is set to 'custom'
+    if model_list == 'custom':
+        v2 = gr.Checkbox.update(visible=True)
+        v_parameterization = gr.Checkbox.update(visible=True)
+        sdxl = gr.Checkbox.update(visible=True)
+        pretrained_model_name_or_path = gr.Textbox.update(visible=True)
+        pretrained_model_name_or_path_file = gr.Button.update(visible=True)
+        pretrained_model_name_or_path_folder = gr.Button.update(visible=True)
+        return model_list, pretrained_model_name_or_path, pretrained_model_name_or_path_file, pretrained_model_name_or_path_folder, v2, v_parameterization, sdxl
+
+
+###
+### Gradio common GUI section
+###
+ 
+def get_pretrained_model_name_or_path_file(
+    model_list, pretrained_model_name_or_path
+):
+    pretrained_model_name_or_path = get_any_file_path(
+        pretrained_model_name_or_path
+    )
+    # set_model_list(model_list, pretrained_model_name_or_path)
+
+
+def get_int_or_default(kwargs, key, default_value=0):
+    value = kwargs.get(key, default_value)
+    if isinstance(value, int):
+        return value
+    elif isinstance(value, str):
+        return int(value)
+    elif isinstance(value, float):
+        return int(value)
+    else:
+        log.info(f'{key} is not an int, float or a string, setting value to {default_value}')
+        return default_value
+    
+def get_float_or_default(kwargs, key, default_value=0.0):
+    value = kwargs.get(key, default_value)
+    if isinstance(value, float):
+        return value
+    elif isinstance(value, int):
+        return float(value)
+    elif isinstance(value, str):
+        return float(value)
+    else:
+        log.info(f'{key} is not an int, float or a string, setting value to {default_value}')
+        return default_value
+
+def get_str_or_default(kwargs, key, default_value=""):
+    value = kwargs.get(key, default_value)
+    if isinstance(value, str):
+        return value
+    elif isinstance(value, int):
+        return str(value)
+    elif isinstance(value, str):
+        return str(value)
+    else:
+        return default_value
+
+def run_cmd_training(**kwargs):
+    run_cmd = ''
+    
+    learning_rate = kwargs.get("learning_rate", "")
+    if learning_rate:
+        run_cmd += f' --learning_rate="{learning_rate}"'
+    
+    lr_scheduler = kwargs.get("lr_scheduler", "")
+    if lr_scheduler:
+        run_cmd += f' --lr_scheduler="{lr_scheduler}"'
+    
+    lr_warmup_steps = kwargs.get("lr_warmup_steps", "")
+    if lr_warmup_steps:
+        if lr_scheduler == 'constant':
+            log.info('Can\'t use LR warmup with LR Scheduler constant... ignoring...')
+        else:
+            run_cmd += f' --lr_warmup_steps="{lr_warmup_steps}"'
+    
+    train_batch_size = kwargs.get("train_batch_size", "")
+    if train_batch_size:
+        run_cmd += f' --train_batch_size="{train_batch_size}"'
+    
+    max_train_steps = kwargs.get("max_train_steps", "")
+    if max_train_steps:
+        run_cmd += f' --max_train_steps="{max_train_steps}"'
+    
+    save_every_n_epochs = kwargs.get("save_every_n_epochs")
+    if save_every_n_epochs:
+        run_cmd += f' --save_every_n_epochs="{int(save_every_n_epochs)}"'
+    
+    mixed_precision = kwargs.get("mixed_precision", "")
+    if mixed_precision:
+        run_cmd += f' --mixed_precision="{mixed_precision}"'
+    
+    save_precision = kwargs.get("save_precision", "")
+    if save_precision:
+        run_cmd += f' --save_precision="{save_precision}"'
+    
+    seed = kwargs.get("seed", "")
+    if seed != '':
+        run_cmd += f' --seed="{seed}"'
+    
+    caption_extension = kwargs.get("caption_extension", "")
+    if caption_extension:
+        run_cmd += f' --caption_extension="{caption_extension}"'
+    
+    cache_latents = kwargs.get('cache_latents')
+    if cache_latents:
+        run_cmd += ' --cache_latents'
+    
+    cache_latents_to_disk = kwargs.get('cache_latents_to_disk')
+    if cache_latents_to_disk:
+        run_cmd += ' --cache_latents_to_disk'
+    
+    optimizer_type = kwargs.get("optimizer", "AdamW")
+    run_cmd += f' --optimizer_type="{optimizer_type}"'
+    
+    optimizer_args = kwargs.get("optimizer_args", "")
+    if optimizer_args != '':
+        run_cmd += f' --optimizer_args {optimizer_args}'
+    
+    return run_cmd
+
+
+def run_cmd_advanced_training(**kwargs):
+    run_cmd = ''
+    
+    max_train_epochs = kwargs.get("max_train_epochs", "")
+    if max_train_epochs:
+        run_cmd += f' --max_train_epochs={max_train_epochs}'
+        
+    max_data_loader_n_workers = kwargs.get("max_data_loader_n_workers", "")
+    if max_data_loader_n_workers:
+        run_cmd += f' --max_data_loader_n_workers="{max_data_loader_n_workers}"'
+    
+    max_token_length = int(kwargs.get("max_token_length", 75))
+    if max_token_length > 75:
+        run_cmd += f' --max_token_length={max_token_length}'
+        
+    clip_skip = int(kwargs.get("clip_skip", 1))
+    if clip_skip > 1:
+        run_cmd += f' --clip_skip={clip_skip}'
+        
+    resume = kwargs.get("resume", "")
+    if resume:
+        run_cmd += f' --resume="{resume}"'
+        
+    keep_tokens = int(kwargs.get("keep_tokens", 0))
+    if keep_tokens > 0:
+        run_cmd += f' --keep_tokens="{keep_tokens}"'
+        
+    caption_dropout_every_n_epochs = int(kwargs.get("caption_dropout_every_n_epochs", 0))
+    if caption_dropout_every_n_epochs > 0:
+        run_cmd += f' --caption_dropout_every_n_epochs="{caption_dropout_every_n_epochs}"'
+    
+    caption_dropout_rate = float(kwargs.get("caption_dropout_rate", 0))
+    if caption_dropout_rate > 0:
+        run_cmd += f' --caption_dropout_rate="{caption_dropout_rate}"'
+        
+    vae_batch_size = int(kwargs.get("vae_batch_size", 0))
+    if vae_batch_size > 0:
+        run_cmd += f' --vae_batch_size="{vae_batch_size}"'
+        
+    bucket_reso_steps = int(kwargs.get("bucket_reso_steps", 64))
+    run_cmd += f' --bucket_reso_steps={bucket_reso_steps}'
+        
+    save_every_n_steps = int(kwargs.get("save_every_n_steps", 0))
+    if save_every_n_steps > 0:
+        run_cmd += f' --save_every_n_steps="{save_every_n_steps}"'
+        
+    save_last_n_steps = int(kwargs.get("save_last_n_steps", 0))
+    if save_last_n_steps > 0:
+        run_cmd += f' --save_last_n_steps="{save_last_n_steps}"'
+        
+    save_last_n_steps_state = int(kwargs.get("save_last_n_steps_state", 0))
+    if save_last_n_steps_state > 0:
+        run_cmd += f' --save_last_n_steps_state="{save_last_n_steps_state}"'
+        
+    min_snr_gamma = int(kwargs.get("min_snr_gamma", 0))
+    if min_snr_gamma >= 1:
+        run_cmd += f' --min_snr_gamma={min_snr_gamma}'
+        
+    min_timestep = int(kwargs.get("min_timestep", 0))
+    if min_timestep > 0:
+        run_cmd += f' --min_timestep={min_timestep}'
+        
+    max_timestep = int(kwargs.get("max_timestep", 1000))
+    if max_timestep < 1000:
+        run_cmd += f' --max_timestep={max_timestep}'
+    
+    save_state = kwargs.get('save_state')
+    if save_state:
+        run_cmd += ' --save_state'
+        
+    mem_eff_attn = kwargs.get('mem_eff_attn')
+    if mem_eff_attn:
+        run_cmd += ' --mem_eff_attn'
+    
+    color_aug = kwargs.get('color_aug')
+    if color_aug:
+        run_cmd += ' --color_aug'
+    
+    flip_aug = kwargs.get('flip_aug')
+    if flip_aug:
+        run_cmd += ' --flip_aug'
+    
+    shuffle_caption = kwargs.get('shuffle_caption')
+    if shuffle_caption:
+        run_cmd += ' --shuffle_caption'
+    
+    gradient_checkpointing = kwargs.get('gradient_checkpointing')
+    if gradient_checkpointing:
+        run_cmd += ' --gradient_checkpointing'
+    
+    full_fp16 = kwargs.get('full_fp16')
+    if full_fp16:
+        run_cmd += ' --full_fp16'
+    
+    xformers = kwargs.get('xformers')
+    if xformers:
+        run_cmd += ' --xformers'
+    
+    persistent_data_loader_workers = kwargs.get('persistent_data_loader_workers')
+    if persistent_data_loader_workers:
+        run_cmd += ' --persistent_data_loader_workers'
+    
+    bucket_no_upscale = kwargs.get('bucket_no_upscale')
+    if bucket_no_upscale:
+        run_cmd += ' --bucket_no_upscale'
+    
+    random_crop = kwargs.get('random_crop')
+    if random_crop:
+        run_cmd += ' --random_crop'
+        
+    scale_v_pred_loss_like_noise_pred = kwargs.get('scale_v_pred_loss_like_noise_pred')
+    if scale_v_pred_loss_like_noise_pred:
+        run_cmd += ' --scale_v_pred_loss_like_noise_pred'
+        
+    noise_offset_type = kwargs.get('noise_offset_type', 'Original')
+    if noise_offset_type == 'Original':
+        noise_offset = float(kwargs.get("noise_offset", 0))
+        if noise_offset > 0:
+            run_cmd += f' --noise_offset={noise_offset}'
+        
+        adaptive_noise_scale = float(kwargs.get("adaptive_noise_scale", 0))
+        if adaptive_noise_scale != 0 and noise_offset > 0:
+            run_cmd += f' --adaptive_noise_scale={adaptive_noise_scale}'
+    else:
+        multires_noise_iterations = int(kwargs.get("multires_noise_iterations", 0))
+        if multires_noise_iterations > 0:
+            run_cmd += f' --multires_noise_iterations="{multires_noise_iterations}"'
+        
+        multires_noise_discount = float(kwargs.get("multires_noise_discount", 0))
+        if multires_noise_discount > 0:
+            run_cmd += f' --multires_noise_discount="{multires_noise_discount}"'
+    
+    additional_parameters = kwargs.get("additional_parameters", "")
+    if additional_parameters:
+        run_cmd += f' {additional_parameters}'
+    
+    use_wandb = kwargs.get('use_wandb')
+    if use_wandb:
+        run_cmd += ' --log_with wandb'
+    
+    wandb_api_key = kwargs.get("wandb_api_key", "")
+    if wandb_api_key:
+        run_cmd += f' --wandb_api_key="{wandb_api_key}"'
+        
+    return run_cmd
+
+def verify_image_folder_pattern(folder_path):
+    false_response = True # temporarily set to true to prevent stopping training in case of false positive
+    true_response = True
+
+    # Check if the folder exists
+    if not os.path.isdir(folder_path):
+        log.error(f"The provided path '{folder_path}' is not a valid folder. Please follow the folder structure documentation found at docs\image_folder_structure.md ...")
+        return false_response
+
+    # Create a regular expression pattern to match the required sub-folder names
+    # The pattern should start with one or more digits (\d+) followed by an underscore (_)
+    # After the underscore, it should match one or more word characters (\w+), which can be letters, numbers, or underscores
+    # Example of a valid pattern matching name: 123_example_folder
+    pattern = r'^\d+_\w+'
+
+    # Get the list of sub-folders in the directory
+    subfolders = [
+        os.path.join(folder_path, subfolder)
+        for subfolder in os.listdir(folder_path)
+        if os.path.isdir(os.path.join(folder_path, subfolder))
+    ]
+
+    # Check the pattern of each sub-folder
+    matching_subfolders = [subfolder for subfolder in subfolders if re.match(pattern, os.path.basename(subfolder))]
+
+    # Print non-matching sub-folders
+    non_matching_subfolders = set(subfolders) - set(matching_subfolders)
+    if non_matching_subfolders:
+        log.error(f"The following folders do not match the required pattern <number>_<text>: {', '.join(non_matching_subfolders)}")
+        log.error(f"Please follow the folder structure documentation found at docs\image_folder_structure.md ...")
+        return false_response
+
+    # Check if no sub-folders exist
+    if not matching_subfolders:
+        log.error(f"No image folders found in {folder_path}. Please follow the folder structure documentation found at docs\image_folder_structure.md ...")
+        return false_response
+
+    log.info(f'Valid image folder names found in: {folder_path}')
+    return true_response
+
+def SaveConfigFile(parameters, file_path: str, exclusion = ['file_path', 'save_as', 'headless', 'print_only']):
+    # Return the values of the variables as a dictionary
+    variables = {
+        name: value
+        for name, value in sorted(parameters, key=lambda x: x[0])
+        if name not in exclusion
+    }
+
+    # Save the data to the selected file
+    with open(file_path, 'w') as file:
+        json.dump(variables, file, indent=2)
+        
+def save_to_file(content):
+    file_path = 'logs/print_command.txt'
+    with open(file_path, 'a') as file:
+        file.write(content + '\n')
\ No newline at end of file
diff --git a/library/config_util.py b/library/config_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..3604ea576b4060139c1d731eb26f33c4411b8492
--- /dev/null
+++ b/library/config_util.py
@@ -0,0 +1,615 @@
+import argparse
+from dataclasses import (
+  asdict,
+  dataclass,
+)
+import functools
+import random
+from textwrap import dedent, indent
+import json
+from pathlib import Path
+# from toolz import curry
+from typing import (
+  List,
+  Optional,
+  Sequence,
+  Tuple,
+  Union,
+)
+
+import toml
+import voluptuous
+from voluptuous import (
+  Any,
+  ExactSequence,
+  MultipleInvalid,
+  Object,
+  Required,
+  Schema,
+)
+from transformers import CLIPTokenizer
+
+from . import train_util
+from .train_util import (
+  DreamBoothSubset,
+  FineTuningSubset,
+  ControlNetSubset,
+  DreamBoothDataset,
+  FineTuningDataset,
+  ControlNetDataset,
+  DatasetGroup,
+)
+
+
+def add_config_arguments(parser: argparse.ArgumentParser):
+  parser.add_argument("--dataset_config", type=Path, default=None, help="config file for detail settings / 詳細な設定用の設定ファイル")
+
+# TODO: inherit Params class in Subset, Dataset
+
+@dataclass
+class BaseSubsetParams:
+  image_dir: Optional[str] = None
+  num_repeats: int = 1
+  shuffle_caption: bool = False
+  keep_tokens: int = 0
+  color_aug: bool = False
+  flip_aug: bool = False
+  face_crop_aug_range: Optional[Tuple[float, float]] = None
+  random_crop: bool = False
+  caption_dropout_rate: float = 0.0
+  caption_dropout_every_n_epochs: int = 0
+  caption_tag_dropout_rate: float = 0.0
+  token_warmup_min: int = 1
+  token_warmup_step: float = 0
+
+@dataclass
+class DreamBoothSubsetParams(BaseSubsetParams):
+  is_reg: bool = False
+  class_tokens: Optional[str] = None
+  caption_extension: str = ".caption"
+
+@dataclass
+class FineTuningSubsetParams(BaseSubsetParams):
+  metadata_file: Optional[str] = None
+
+@dataclass
+class ControlNetSubsetParams(BaseSubsetParams):
+  conditioning_data_dir: str = None
+  caption_extension: str = ".caption"
+
+@dataclass
+class BaseDatasetParams:
+  tokenizer: Union[CLIPTokenizer, List[CLIPTokenizer]] = None
+  max_token_length: int = None
+  resolution: Optional[Tuple[int, int]] = None
+  debug_dataset: bool = False
+
+@dataclass
+class DreamBoothDatasetParams(BaseDatasetParams):
+  batch_size: int = 1
+  enable_bucket: bool = False
+  min_bucket_reso: int = 256
+  max_bucket_reso: int = 1024
+  bucket_reso_steps: int = 64
+  bucket_no_upscale: bool = False
+  prior_loss_weight: float = 1.0
+
+@dataclass
+class FineTuningDatasetParams(BaseDatasetParams):
+  batch_size: int = 1
+  enable_bucket: bool = False
+  min_bucket_reso: int = 256
+  max_bucket_reso: int = 1024
+  bucket_reso_steps: int = 64
+  bucket_no_upscale: bool = False
+
+@dataclass
+class ControlNetDatasetParams(BaseDatasetParams):
+  batch_size: int = 1
+  enable_bucket: bool = False
+  min_bucket_reso: int = 256
+  max_bucket_reso: int = 1024
+  bucket_reso_steps: int = 64
+  bucket_no_upscale: bool = False
+
+@dataclass
+class SubsetBlueprint:
+  params: Union[DreamBoothSubsetParams, FineTuningSubsetParams]
+
+@dataclass
+class DatasetBlueprint:
+  is_dreambooth: bool
+  is_controlnet: bool
+  params: Union[DreamBoothDatasetParams, FineTuningDatasetParams]
+  subsets: Sequence[SubsetBlueprint]
+
+@dataclass
+class DatasetGroupBlueprint:
+  datasets: Sequence[DatasetBlueprint]
+@dataclass
+class Blueprint:
+  dataset_group: DatasetGroupBlueprint
+
+
+class ConfigSanitizer:
+  # @curry
+  @staticmethod
+  def __validate_and_convert_twodim(klass, value: Sequence) -> Tuple:
+    Schema(ExactSequence([klass, klass]))(value)
+    return tuple(value)
+
+  # @curry
+  @staticmethod
+  def __validate_and_convert_scalar_or_twodim(klass, value: Union[float, Sequence]) -> Tuple:
+    Schema(Any(klass, ExactSequence([klass, klass])))(value)
+    try:
+      Schema(klass)(value)
+      return (value, value)
+    except:
+      return ConfigSanitizer.__validate_and_convert_twodim(klass, value)
+
+  # subset schema
+  SUBSET_ASCENDABLE_SCHEMA = {
+    "color_aug": bool,
+    "face_crop_aug_range": functools.partial(__validate_and_convert_twodim.__func__, float),
+    "flip_aug": bool,
+    "num_repeats": int,
+    "random_crop": bool,
+    "shuffle_caption": bool,
+    "keep_tokens": int,
+    "token_warmup_min": int,
+    "token_warmup_step": Any(float,int),
+  }
+  # DO means DropOut
+  DO_SUBSET_ASCENDABLE_SCHEMA = {
+    "caption_dropout_every_n_epochs": int,
+    "caption_dropout_rate": Any(float, int),
+    "caption_tag_dropout_rate": Any(float, int),
+  }
+  # DB means DreamBooth
+  DB_SUBSET_ASCENDABLE_SCHEMA = {
+    "caption_extension": str,
+    "class_tokens": str,
+  }
+  DB_SUBSET_DISTINCT_SCHEMA = {
+    Required("image_dir"): str,
+    "is_reg": bool,
+  }
+  # FT means FineTuning
+  FT_SUBSET_DISTINCT_SCHEMA = {
+    Required("metadata_file"): str,
+    "image_dir": str,
+  }
+  CN_SUBSET_ASCENDABLE_SCHEMA = {
+    "caption_extension": str,
+  }
+  CN_SUBSET_DISTINCT_SCHEMA = {
+    Required("image_dir"): str,
+    Required("conditioning_data_dir"): str,
+  }
+
+  # datasets schema
+  DATASET_ASCENDABLE_SCHEMA = {
+    "batch_size": int,
+    "bucket_no_upscale": bool,
+    "bucket_reso_steps": int,
+    "enable_bucket": bool,
+    "max_bucket_reso": int,
+    "min_bucket_reso": int,
+    "resolution": functools.partial(__validate_and_convert_scalar_or_twodim.__func__, int),
+  }
+
+  # options handled by argparse but not handled by user config
+  ARGPARSE_SPECIFIC_SCHEMA = {
+    "debug_dataset": bool,
+    "max_token_length": Any(None, int),
+    "prior_loss_weight": Any(float, int),
+  }
+  # for handling default None value of argparse
+  ARGPARSE_NULLABLE_OPTNAMES = [
+    "face_crop_aug_range",
+    "resolution",
+  ]
+  # prepare map because option name may differ among argparse and user config
+  ARGPARSE_OPTNAME_TO_CONFIG_OPTNAME = {
+    "train_batch_size": "batch_size",
+    "dataset_repeats": "num_repeats",
+  }
+
+  def __init__(self, support_dreambooth: bool, support_finetuning: bool, support_controlnet: bool, support_dropout: bool) -> None:
+    assert support_dreambooth or support_finetuning or support_controlnet, "Neither DreamBooth mode nor fine tuning mode specified. Please specify one mode or more. / DreamBooth モードか fine tuning モードのどちらも指定されていません。1つ以上指定してください。"
+
+    self.db_subset_schema = self.__merge_dict(
+      self.SUBSET_ASCENDABLE_SCHEMA,
+      self.DB_SUBSET_DISTINCT_SCHEMA,
+      self.DB_SUBSET_ASCENDABLE_SCHEMA,
+      self.DO_SUBSET_ASCENDABLE_SCHEMA if support_dropout else {},
+    )
+
+    self.ft_subset_schema = self.__merge_dict(
+      self.SUBSET_ASCENDABLE_SCHEMA,
+      self.FT_SUBSET_DISTINCT_SCHEMA,
+      self.DO_SUBSET_ASCENDABLE_SCHEMA if support_dropout else {},
+    )
+
+    self.cn_subset_schema = self.__merge_dict(
+      self.SUBSET_ASCENDABLE_SCHEMA,
+      self.CN_SUBSET_DISTINCT_SCHEMA,
+      self.CN_SUBSET_ASCENDABLE_SCHEMA,
+      self.DO_SUBSET_ASCENDABLE_SCHEMA if support_dropout else {},
+    )
+
+    self.db_dataset_schema = self.__merge_dict(
+      self.DATASET_ASCENDABLE_SCHEMA,
+      self.SUBSET_ASCENDABLE_SCHEMA,
+      self.DB_SUBSET_ASCENDABLE_SCHEMA,
+      self.DO_SUBSET_ASCENDABLE_SCHEMA if support_dropout else {},
+      {"subsets": [self.db_subset_schema]},
+    )
+
+    self.ft_dataset_schema = self.__merge_dict(
+      self.DATASET_ASCENDABLE_SCHEMA,
+      self.SUBSET_ASCENDABLE_SCHEMA,
+      self.DO_SUBSET_ASCENDABLE_SCHEMA if support_dropout else {},
+      {"subsets": [self.ft_subset_schema]},
+    )
+
+    self.cn_dataset_schema = self.__merge_dict(
+      self.DATASET_ASCENDABLE_SCHEMA,
+      self.SUBSET_ASCENDABLE_SCHEMA,
+      self.CN_SUBSET_ASCENDABLE_SCHEMA,
+      self.DO_SUBSET_ASCENDABLE_SCHEMA if support_dropout else {},
+      {"subsets": [self.cn_subset_schema]},
+    )
+
+    if support_dreambooth and support_finetuning:
+      def validate_flex_dataset(dataset_config: dict):
+        subsets_config = dataset_config.get("subsets", [])
+
+        if support_controlnet and all(["conditioning_data_dir" in subset for subset in subsets_config]):
+          return Schema(self.cn_dataset_schema)(dataset_config)
+        # check dataset meets FT style
+        # NOTE: all FT subsets should have "metadata_file"
+        elif all(["metadata_file" in subset for subset in subsets_config]):
+          return Schema(self.ft_dataset_schema)(dataset_config)
+        # check dataset meets DB style
+        # NOTE: all DB subsets should have no "metadata_file"
+        elif all(["metadata_file" not in subset for subset in subsets_config]):
+          return Schema(self.db_dataset_schema)(dataset_config)
+        else:
+          raise voluptuous.Invalid("DreamBooth subset and fine tuning subset cannot be mixed in the same dataset. Please split them into separate datasets. / DreamBoothのサブセットとfine tuninのサブセットを同一のデータセットに混在させることはできません。別々のデータセットに分割してください。")
+
+      self.dataset_schema = validate_flex_dataset
+    elif support_dreambooth:
+      self.dataset_schema = self.db_dataset_schema
+    elif support_finetuning:
+      self.dataset_schema = self.ft_dataset_schema
+    elif support_controlnet:
+      self.dataset_schema = self.cn_dataset_schema
+
+    self.general_schema = self.__merge_dict(
+      self.DATASET_ASCENDABLE_SCHEMA,
+      self.SUBSET_ASCENDABLE_SCHEMA,
+      self.DB_SUBSET_ASCENDABLE_SCHEMA if support_dreambooth else {},
+      self.CN_SUBSET_ASCENDABLE_SCHEMA if support_controlnet else {},
+      self.DO_SUBSET_ASCENDABLE_SCHEMA if support_dropout else {},
+    )
+
+    self.user_config_validator = Schema({
+      "general": self.general_schema,
+      "datasets": [self.dataset_schema],
+    })
+
+    self.argparse_schema = self.__merge_dict(
+      self.general_schema,
+      self.ARGPARSE_SPECIFIC_SCHEMA,
+      {optname: Any(None, self.general_schema[optname]) for optname in self.ARGPARSE_NULLABLE_OPTNAMES},
+      {a_name: self.general_schema[c_name] for a_name, c_name in self.ARGPARSE_OPTNAME_TO_CONFIG_OPTNAME.items()},
+    )
+
+    self.argparse_config_validator = Schema(Object(self.argparse_schema), extra=voluptuous.ALLOW_EXTRA)
+
+  def sanitize_user_config(self, user_config: dict) -> dict:
+    try:
+      return self.user_config_validator(user_config)
+    except MultipleInvalid:
+      # TODO: エラー発生時のメッセージをわかりやすくする
+      print("Invalid user config / ユーザ設定の形式が正しくないようです")
+      raise
+
+  # NOTE: In nature, argument parser result is not needed to be sanitize
+  #   However this will help us to detect program bug
+  def sanitize_argparse_namespace(self, argparse_namespace: argparse.Namespace) -> argparse.Namespace:
+    try:
+      return self.argparse_config_validator(argparse_namespace)
+    except MultipleInvalid:
+      # XXX: this should be a bug
+      print("Invalid cmdline parsed arguments. This should be a bug. / コマンドラインのパース結果が正しくないようです。プログラムのバグの可能性が高いです。")
+      raise
+
+  # NOTE: value would be overwritten by latter dict if there is already the same key
+  @staticmethod
+  def __merge_dict(*dict_list: dict) -> dict:
+    merged = {}
+    for schema in dict_list:
+      # merged |= schema
+      for k, v in schema.items():
+        merged[k] = v
+    return merged
+
+
+class BlueprintGenerator:
+  BLUEPRINT_PARAM_NAME_TO_CONFIG_OPTNAME = {
+  }
+
+  def __init__(self, sanitizer: ConfigSanitizer):
+    self.sanitizer = sanitizer
+
+  # runtime_params is for parameters which is only configurable on runtime, such as tokenizer
+  def generate(self, user_config: dict, argparse_namespace: argparse.Namespace, **runtime_params) -> Blueprint:
+    sanitized_user_config = self.sanitizer.sanitize_user_config(user_config)
+    sanitized_argparse_namespace = self.sanitizer.sanitize_argparse_namespace(argparse_namespace)
+
+    # convert argparse namespace to dict like config
+    # NOTE: it is ok to have extra entries in dict
+    optname_map = self.sanitizer.ARGPARSE_OPTNAME_TO_CONFIG_OPTNAME
+    argparse_config = {optname_map.get(optname, optname): value for optname, value in vars(sanitized_argparse_namespace).items()}
+
+    general_config = sanitized_user_config.get("general", {})
+
+    dataset_blueprints = []
+    for dataset_config in sanitized_user_config.get("datasets", []):
+      # NOTE: if subsets have no "metadata_file", these are DreamBooth datasets/subsets
+      subsets = dataset_config.get("subsets", [])
+      is_dreambooth = all(["metadata_file" not in subset for subset in subsets])
+      is_controlnet = all(["conditioning_data_dir" in subset for subset in subsets])
+      if is_controlnet:
+        subset_params_klass = ControlNetSubsetParams
+        dataset_params_klass = ControlNetDatasetParams
+      elif is_dreambooth:
+        subset_params_klass = DreamBoothSubsetParams
+        dataset_params_klass = DreamBoothDatasetParams
+      else:
+        subset_params_klass = FineTuningSubsetParams
+        dataset_params_klass = FineTuningDatasetParams
+
+      subset_blueprints = []
+      for subset_config in subsets:
+        params = self.generate_params_by_fallbacks(subset_params_klass,
+                                                   [subset_config, dataset_config, general_config, argparse_config, runtime_params])
+        subset_blueprints.append(SubsetBlueprint(params))
+
+      params = self.generate_params_by_fallbacks(dataset_params_klass,
+                                                 [dataset_config, general_config, argparse_config, runtime_params])
+      dataset_blueprints.append(DatasetBlueprint(is_dreambooth, is_controlnet, params, subset_blueprints))
+
+    dataset_group_blueprint = DatasetGroupBlueprint(dataset_blueprints)
+
+    return Blueprint(dataset_group_blueprint)
+
+  @staticmethod
+  def generate_params_by_fallbacks(param_klass, fallbacks: Sequence[dict]):
+    name_map = BlueprintGenerator.BLUEPRINT_PARAM_NAME_TO_CONFIG_OPTNAME
+    search_value = BlueprintGenerator.search_value
+    default_params = asdict(param_klass())
+    param_names = default_params.keys()
+
+    params = {name: search_value(name_map.get(name, name), fallbacks, default_params.get(name)) for name in param_names}
+
+    return param_klass(**params)
+
+  @staticmethod
+  def search_value(key: str, fallbacks: Sequence[dict], default_value = None):
+    for cand in fallbacks:
+      value = cand.get(key)
+      if value is not None:
+        return value
+
+    return default_value
+
+
+def generate_dataset_group_by_blueprint(dataset_group_blueprint: DatasetGroupBlueprint):
+  datasets: List[Union[DreamBoothDataset, FineTuningDataset, ControlNetDataset]] = []
+
+  for dataset_blueprint in dataset_group_blueprint.datasets:
+    if dataset_blueprint.is_controlnet:
+      subset_klass = ControlNetSubset
+      dataset_klass = ControlNetDataset
+    elif dataset_blueprint.is_dreambooth:
+      subset_klass = DreamBoothSubset
+      dataset_klass = DreamBoothDataset
+    else:
+      subset_klass = FineTuningSubset
+      dataset_klass = FineTuningDataset
+
+    subsets = [subset_klass(**asdict(subset_blueprint.params)) for subset_blueprint in dataset_blueprint.subsets]
+    dataset = dataset_klass(subsets=subsets, **asdict(dataset_blueprint.params))
+    datasets.append(dataset)
+
+  # print info
+  info = ""
+  for i, dataset in enumerate(datasets):
+    is_dreambooth = isinstance(dataset, DreamBoothDataset)
+    is_controlnet = isinstance(dataset, ControlNetDataset)
+    info += dedent(f"""\
+      [Dataset {i}]
+        batch_size: {dataset.batch_size}
+        resolution: {(dataset.width, dataset.height)}
+        enable_bucket: {dataset.enable_bucket}
+    """)
+
+    if dataset.enable_bucket:
+      info += indent(dedent(f"""\
+        min_bucket_reso: {dataset.min_bucket_reso}
+        max_bucket_reso: {dataset.max_bucket_reso}
+        bucket_reso_steps: {dataset.bucket_reso_steps}
+        bucket_no_upscale: {dataset.bucket_no_upscale}
+      \n"""), "  ")
+    else:
+      info += "\n"
+
+    for j, subset in enumerate(dataset.subsets):
+      info += indent(dedent(f"""\
+        [Subset {j} of Dataset {i}]
+          image_dir: "{subset.image_dir}"
+          image_count: {subset.img_count}
+          num_repeats: {subset.num_repeats}
+          shuffle_caption: {subset.shuffle_caption}
+          keep_tokens: {subset.keep_tokens}
+          caption_dropout_rate: {subset.caption_dropout_rate}
+          caption_dropout_every_n_epoches: {subset.caption_dropout_every_n_epochs}
+          caption_tag_dropout_rate: {subset.caption_tag_dropout_rate}
+          color_aug: {subset.color_aug}
+          flip_aug: {subset.flip_aug}
+          face_crop_aug_range: {subset.face_crop_aug_range}
+          random_crop: {subset.random_crop}
+          token_warmup_min: {subset.token_warmup_min},
+          token_warmup_step: {subset.token_warmup_step},
+      """), "  ")
+
+      if is_dreambooth:
+        info += indent(dedent(f"""\
+          is_reg: {subset.is_reg}
+          class_tokens: {subset.class_tokens}
+          caption_extension: {subset.caption_extension}
+        \n"""), "    ")
+      elif not is_controlnet:
+        info += indent(dedent(f"""\
+          metadata_file: {subset.metadata_file}
+        \n"""), "    ")
+
+  print(info)
+
+  # make buckets first because it determines the length of dataset
+  # and set the same seed for all datasets
+  seed = random.randint(0, 2**31) # actual seed is seed + epoch_no
+  for i, dataset in enumerate(datasets):
+    print(f"[Dataset {i}]")
+    dataset.make_buckets()
+    dataset.set_seed(seed)
+
+  return DatasetGroup(datasets)
+
+
+def generate_dreambooth_subsets_config_by_subdirs(train_data_dir: Optional[str] = None, reg_data_dir: Optional[str] = None):
+  def extract_dreambooth_params(name: str) -> Tuple[int, str]:
+    tokens = name.split('_')
+    try:
+      n_repeats = int(tokens[0])
+    except ValueError as e:
+      print(f"ignore directory without repeats / 繰り返し回数のないディレクトリを無視します: {name}")
+      return 0, ""
+    caption_by_folder = '_'.join(tokens[1:])
+    return n_repeats, caption_by_folder
+
+  def generate(base_dir: Optional[str], is_reg: bool):
+    if base_dir is None:
+      return []
+
+    base_dir: Path = Path(base_dir)
+    if not base_dir.is_dir():
+      return []
+
+    subsets_config = []
+    for subdir in base_dir.iterdir():
+      if not subdir.is_dir():
+        continue
+
+      num_repeats, class_tokens = extract_dreambooth_params(subdir.name)
+      if num_repeats < 1:
+        continue
+
+      subset_config = {"image_dir": str(subdir), "num_repeats": num_repeats, "is_reg": is_reg, "class_tokens": class_tokens}
+      subsets_config.append(subset_config)
+
+    return subsets_config
+
+  subsets_config = []
+  subsets_config += generate(train_data_dir, False)
+  subsets_config += generate(reg_data_dir, True)
+
+  return subsets_config
+
+
+def generate_controlnet_subsets_config_by_subdirs(train_data_dir: Optional[str] = None, conditioning_data_dir: Optional[str] = None, caption_extension: str = ".txt"):
+  def generate(base_dir: Optional[str]):
+    if base_dir is None:
+      return []
+
+    base_dir: Path = Path(base_dir)
+    if not base_dir.is_dir():
+      return []
+
+    subsets_config = []
+    for subdir in base_dir.iterdir():
+      if not subdir.is_dir():
+        continue
+
+      subset_config = {"image_dir": str(subdir), "conditioning_data_dir": conditioning_data_dir, "caption_extension": caption_extension, "num_repeats": 1}
+      subsets_config.append(subset_config)
+
+    return subsets_config
+
+  subsets_config = []
+  subsets_config += generate(train_data_dir, False)
+
+  return subsets_config
+
+
+def load_user_config(file: str) -> dict:
+  file: Path = Path(file)
+  if not file.is_file():
+    raise ValueError(f"file not found / ファイルが見つかりません: {file}")
+
+  if file.name.lower().endswith('.json'):
+    try:
+      with open(file, 'r') as f:
+        config = json.load(f)
+    except Exception:
+      print(f"Error on parsing JSON config file. Please check the format. / JSON 形式の設定ファイルの読み込みに失敗しました。文法が正しいか確認してください。: {file}")
+      raise
+  elif file.name.lower().endswith('.toml'):
+    try:
+      config = toml.load(file)
+    except Exception:
+      print(f"Error on parsing TOML config file. Please check the format. / TOML 形式の設定ファイルの読み込みに失敗しました。文法が正しいか確認してください。: {file}")
+      raise
+  else:
+    raise ValueError(f"not supported config file format / 対応していない設定ファイルの形式です: {file}")
+
+  return config
+
+# for config test
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--support_dreambooth", action="store_true")
+  parser.add_argument("--support_finetuning", action="store_true")
+  parser.add_argument("--support_controlnet", action="store_true")
+  parser.add_argument("--support_dropout", action="store_true")
+  parser.add_argument("dataset_config")
+  config_args, remain = parser.parse_known_args()
+
+  parser = argparse.ArgumentParser()
+  train_util.add_dataset_arguments(parser, config_args.support_dreambooth, config_args.support_finetuning, config_args.support_dropout)
+  train_util.add_training_arguments(parser, config_args.support_dreambooth)
+  argparse_namespace = parser.parse_args(remain)
+  train_util.prepare_dataset_args(argparse_namespace, config_args.support_finetuning)
+
+  print("[argparse_namespace]")
+  print(vars(argparse_namespace))
+
+  user_config = load_user_config(config_args.dataset_config)
+
+  print("\n[user_config]")
+  print(user_config)
+
+  sanitizer = ConfigSanitizer(config_args.support_dreambooth, config_args.support_finetuning, config_args.support_controlnet, config_args.support_dropout)
+  sanitized_user_config = sanitizer.sanitize_user_config(user_config)
+
+  print("\n[sanitized_user_config]")
+  print(sanitized_user_config)
+
+  blueprint = BlueprintGenerator(sanitizer).generate(user_config, argparse_namespace)
+
+  print("\n[blueprint]")
+  print(blueprint)
diff --git a/library/convert_model_gui.py b/library/convert_model_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..739a02ef7b845e24525fc75c9add6f6a529a2c2a
--- /dev/null
+++ b/library/convert_model_gui.py
@@ -0,0 +1,277 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+import shutil
+from .common_gui import get_folder_path, get_file_path
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def convert_model(
+    source_model_input,
+    source_model_type,
+    target_model_folder_input,
+    target_model_name_input,
+    target_model_type,
+    target_save_precision_type,
+    unet_use_linear_projection,
+):
+    # Check for caption_text_input
+    if source_model_type == '':
+        msgbox('Invalid source model type')
+        return
+
+    # Check if source model exist
+    if os.path.isfile(source_model_input):
+        log.info('The provided source model is a file')
+    elif os.path.isdir(source_model_input):
+        log.info('The provided model is a folder')
+    else:
+        msgbox('The provided source model is neither a file nor a folder')
+        return
+
+    # Check if source model exist
+    if os.path.isdir(target_model_folder_input):
+        log.info('The provided model folder exist')
+    else:
+        msgbox('The provided target folder does not exist')
+        return
+
+    run_cmd = f'{PYTHON} "tools/convert_diffusers20_original_sd.py"'
+
+    v1_models = [
+        'runwayml/stable-diffusion-v1-5',
+        'CompVis/stable-diffusion-v1-4',
+    ]
+
+    # check if v1 models
+    if str(source_model_type) in v1_models:
+        log.info('SD v1 model specified. Setting --v1 parameter')
+        run_cmd += ' --v1'
+    else:
+        log.info('SD v2 model specified. Setting --v2 parameter')
+        run_cmd += ' --v2'
+
+    if not target_save_precision_type == 'unspecified':
+        run_cmd += f' --{target_save_precision_type}'
+
+    if (
+        target_model_type == 'diffuser'
+        or target_model_type == 'diffuser_safetensors'
+    ):
+        run_cmd += f' --reference_model="{source_model_type}"'
+
+    if target_model_type == 'diffuser_safetensors':
+        run_cmd += ' --use_safetensors'
+
+    # Fix for stabilityAI diffusers format. When saving v2 models in Diffusers format in training scripts and conversion scripts,
+    # it was found that the U-Net configuration is different from those of Hugging Face's stabilityai models (this repository is
+    # "use_linear_projection": false, stabilityai is true). Please note that the weight shapes are different, so please be careful
+    # when using the weight files directly.
+
+    if unet_use_linear_projection:
+        run_cmd += ' --unet_use_linear_projection'
+
+    run_cmd += f' "{source_model_input}"'
+
+    if (
+        target_model_type == 'diffuser'
+        or target_model_type == 'diffuser_safetensors'
+    ):
+        target_model_path = os.path.join(
+            target_model_folder_input, target_model_name_input
+        )
+        run_cmd += f' "{target_model_path}"'
+    else:
+        target_model_path = os.path.join(
+            target_model_folder_input,
+            f'{target_model_name_input}.{target_model_type}',
+        )
+        run_cmd += f' "{target_model_path}"'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+    if (
+        not target_model_type == 'diffuser'
+        or target_model_type == 'diffuser_safetensors'
+    ):
+
+        v2_models = [
+            'stabilityai/stable-diffusion-2-1-base',
+            'stabilityai/stable-diffusion-2-base',
+        ]
+        v_parameterization = [
+            'stabilityai/stable-diffusion-2-1',
+            'stabilityai/stable-diffusion-2',
+        ]
+
+        if str(source_model_type) in v2_models:
+            inference_file = os.path.join(
+                target_model_folder_input, f'{target_model_name_input}.yaml'
+            )
+            log.info(f'Saving v2-inference.yaml as {inference_file}')
+            shutil.copy(
+                f'./v2_inference/v2-inference.yaml',
+                f'{inference_file}',
+            )
+
+        if str(source_model_type) in v_parameterization:
+            inference_file = os.path.join(
+                target_model_folder_input, f'{target_model_name_input}.yaml'
+            )
+            log.info(f'Saving v2-inference-v.yaml as {inference_file}')
+            shutil.copy(
+                f'./v2_inference/v2-inference-v.yaml',
+                f'{inference_file}',
+            )
+
+
+#   parser = argparse.ArgumentParser()
+#   parser.add_argument("--v1", action='store_true',
+#                       help='load v1.x model (v1 or v2 is required to load checkpoint) / 1.xのモデルを読み込む')
+#   parser.add_argument("--v2", action='store_true',
+#                       help='load v2.0 model (v1 or v2 is required to load checkpoint) / 2.0のモデルを読み込む')
+#   parser.add_argument("--fp16", action='store_true',
+#                       help='load as fp16 (Diffusers only) and save as fp16 (checkpoint only) / fp16形式で読み込み（Diffusers形式のみ対応）、保存する（checkpointのみ対応）')
+#   parser.add_argument("--bf16", action='store_true', help='save as bf16 (checkpoint only) / bf16形式で保存する（checkpointのみ対応）')
+#   parser.add_argument("--float", action='store_true',
+#                       help='save as float (checkpoint only) / float(float32)形式で保存する（checkpointのみ対応）')
+#   parser.add_argument("--epoch", type=int, default=0, help='epoch to write to checkpoint / checkpointに記録するepoch数の値')
+#   parser.add_argument("--global_step", type=int, default=0,
+#                       help='global_step to write to checkpoint / checkpointに記録するglobal_stepの値')
+#   parser.add_argument("--reference_model", type=str, default=None,
+#                       help="reference model for schduler/tokenizer, required in saving Diffusers, copy schduler/tokenizer from this / scheduler/tokenizerのコピー元のDiffusersモデル、Diffusers形式で保存するときに必要")
+
+#   parser.add_argument("model_to_load", type=str, default=None,
+#                       help="model to load: checkpoint file or Diffusers model's directory / 読み込むモデル、checkpointかDiffusers形式モデルのディレクトリ")
+#   parser.add_argument("model_to_save", type=str, default=None,
+#                       help="model to save: checkpoint (with extension) or Diffusers model's directory (without extension) / 変換後のモデル、拡張子がある場合はcheckpoint、ない場合はDiffusesモデルとして保存")
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_convert_model_tab(headless=False):
+    with gr.Tab('Convert model'):
+        gr.Markdown(
+            'This utility can be used to convert from one stable diffusion model format to another.'
+        )
+        
+        model_ext = gr.Textbox(value='*.safetensors *.ckpt', visible=False)
+        model_ext_name = gr.Textbox(value='Model types', visible=False)
+        
+        with gr.Row():
+            source_model_input = gr.Textbox(
+                label='Source model',
+                placeholder='path to source model folder of file to convert...',
+                interactive=True,
+            )
+            button_source_model_dir = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_source_model_dir.click(
+                get_folder_path,
+                outputs=source_model_input,
+                show_progress=False,
+            )
+
+            button_source_model_file = gr.Button(
+                document_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_source_model_file.click(
+                get_file_path,
+                inputs=[source_model_input, model_ext, model_ext_name],
+                outputs=source_model_input,
+                show_progress=False,
+            )
+
+            source_model_type = gr.Dropdown(
+                label='Source model type',
+                choices=[
+                    'stabilityai/stable-diffusion-2-1-base',
+                    'stabilityai/stable-diffusion-2-base',
+                    'stabilityai/stable-diffusion-2-1',
+                    'stabilityai/stable-diffusion-2',
+                    'runwayml/stable-diffusion-v1-5',
+                    'CompVis/stable-diffusion-v1-4',
+                ],
+            )
+        with gr.Row():
+            target_model_folder_input = gr.Textbox(
+                label='Target model folder',
+                placeholder='path to target model folder of file name to create...',
+                interactive=True,
+            )
+            button_target_model_folder = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_target_model_folder.click(
+                get_folder_path,
+                outputs=target_model_folder_input,
+                show_progress=False,
+            )
+
+            target_model_name_input = gr.Textbox(
+                label='Target model name',
+                placeholder='target model name...',
+                interactive=True,
+            )
+            target_model_type = gr.Dropdown(
+                label='Target model type',
+                choices=[
+                    'diffuser',
+                    'diffuser_safetensors',
+                    'ckpt',
+                    'safetensors',
+                ],
+            )
+            target_save_precision_type = gr.Dropdown(
+                label='Target model precision',
+                choices=['unspecified', 'fp16', 'bf16', 'float'],
+                value='unspecified',
+            )
+            unet_use_linear_projection = gr.Checkbox(
+                label='UNet linear projection',
+                value=False,
+                info="Enable for Hugging Face's stabilityai models",
+            )
+
+        convert_button = gr.Button('Convert model')
+
+        convert_button.click(
+            convert_model,
+            inputs=[
+                source_model_input,
+                source_model_type,
+                target_model_folder_input,
+                target_model_name_input,
+                target_model_type,
+                target_save_precision_type,
+                unet_use_linear_projection,
+            ],
+            show_progress=False,
+        )
diff --git a/library/custom_logging.py b/library/custom_logging.py
new file mode 100644
index 0000000000000000000000000000000000000000..ee7e5e208094486dade99f7d7b5207b9512e39c7
--- /dev/null
+++ b/library/custom_logging.py
@@ -0,0 +1,44 @@
+import os
+import logging
+import time
+import sys
+
+from rich.theme import Theme
+from rich.logging import RichHandler
+from rich.console import Console
+from rich.pretty import install as pretty_install
+from rich.traceback import install as traceback_install
+
+log = None
+
+def setup_logging(clean=False, debug=False):
+    global log
+    
+    if log is not None:
+        return log
+    
+    try:
+        if clean and os.path.isfile('setup.log'):
+            os.remove('setup.log')
+        time.sleep(0.1) # prevent race condition
+    except:
+        pass
+    
+    if sys.version_info >= (3, 9):
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', encoding='utf-8', force=True)
+    else:
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', force=True)
+    
+    console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({
+        "traceback.border": "black",
+        "traceback.border.syntax_error": "black",
+        "inspect.value.border": "black",
+    }))
+    pretty_install(console=console)
+    traceback_install(console=console, extra_lines=1, width=console.width, word_wrap=False, indent_guides=False, suppress=[])
+    rh = RichHandler(show_time=True, omit_repeated_times=False, show_level=True, show_path=False, markup=False, rich_tracebacks=True, log_time_format='%H:%M:%S-%f', level=logging.DEBUG if debug else logging.INFO, console=console)
+    rh.set_name(logging.DEBUG if debug else logging.INFO)
+    log = logging.getLogger("sd")
+    log.addHandler(rh)
+    
+    return log
diff --git a/library/custom_train_functions.py b/library/custom_train_functions.py
new file mode 100644
index 0000000000000000000000000000000000000000..6672ad48a08a27ae9d0b1fd2f0f4e2fcf60fa259
--- /dev/null
+++ b/library/custom_train_functions.py
@@ -0,0 +1,458 @@
+import torch
+import argparse
+import random
+import re
+from typing import List, Optional, Union
+
+
+def prepare_scheduler_for_custom_training(noise_scheduler, device):
+    if hasattr(noise_scheduler, "all_snr"):
+        return
+
+    alphas_cumprod = noise_scheduler.alphas_cumprod
+    sqrt_alphas_cumprod = torch.sqrt(alphas_cumprod)
+    sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 - alphas_cumprod)
+    alpha = sqrt_alphas_cumprod
+    sigma = sqrt_one_minus_alphas_cumprod
+    all_snr = (alpha / sigma) ** 2
+
+    noise_scheduler.all_snr = all_snr.to(device)
+
+
+def apply_snr_weight(loss, timesteps, noise_scheduler, gamma):
+    snr = torch.stack([noise_scheduler.all_snr[t] for t in timesteps])
+    gamma_over_snr = torch.div(torch.ones_like(snr) * gamma, snr)
+    snr_weight = torch.minimum(gamma_over_snr, torch.ones_like(gamma_over_snr)).float().to(loss.device)  # from paper
+    loss = loss * snr_weight
+    return loss
+
+
+def scale_v_prediction_loss_like_noise_prediction(loss, timesteps, noise_scheduler):
+    snr_t = torch.stack([noise_scheduler.all_snr[t] for t in timesteps])  # batch_size
+    snr_t = torch.minimum(snr_t, torch.ones_like(snr_t) * 1000)  # if timestep is 0, snr_t is inf, so limit it to 1000
+    scale = snr_t / (snr_t + 1)
+
+    loss = loss * scale
+    return loss
+
+
+# TODO train_utilと分散しているのでどちらかに寄せる
+
+
+def add_custom_train_arguments(parser: argparse.ArgumentParser, support_weighted_captions: bool = True):
+    parser.add_argument(
+        "--min_snr_gamma",
+        type=float,
+        default=None,
+        help="gamma for reducing the weight of high loss timesteps. Lower numbers have stronger effect. 5 is recommended by paper. / 低いタイムステップでの高いlossに対して重みを減らすためのgamma値、低いほど効果が強く、論文では5が推奨",
+    )
+    parser.add_argument(
+        "--scale_v_pred_loss_like_noise_pred",
+        action="store_true",
+        help="scale v-prediction loss like noise prediction loss / v-prediction lossをnoise prediction lossと同じようにスケーリングする",
+    )
+    if support_weighted_captions:
+        parser.add_argument(
+            "--weighted_captions",
+            action="store_true",
+            default=False,
+            help="Enable weighted captions in the standard style (token:1.3). No commas inside parens, or shuffle/dropout may break the decoder. / 「[token]」、「(token)」「(token:1.3)」のような重み付きキャプションを有効にする。カンマを括弧内に入れるとシャッフルやdropoutで重みづけがおかしくなるので注意",
+        )
+
+
+re_attention = re.compile(
+    r"""
+\\\(|
+\\\)|
+\\\[|
+\\]|
+\\\\|
+\\|
+\(|
+\[|
+:([+-]?[.\d]+)\)|
+\)|
+]|
+[^\\()\[\]:]+|
+:
+""",
+    re.X,
+)
+
+
+def parse_prompt_attention(text):
+    """
+    Parses a string with attention tokens and returns a list of pairs: text and its associated weight.
+    Accepted tokens are:
+      (abc) - increases attention to abc by a multiplier of 1.1
+      (abc:3.12) - increases attention to abc by a multiplier of 3.12
+      [abc] - decreases attention to abc by a multiplier of 1.1
+      \( - literal character '('
+      \[ - literal character '['
+      \) - literal character ')'
+      \] - literal character ']'
+      \\ - literal character '\'
+      anything else - just text
+    >>> parse_prompt_attention('normal text')
+    [['normal text', 1.0]]
+    >>> parse_prompt_attention('an (important) word')
+    [['an ', 1.0], ['important', 1.1], [' word', 1.0]]
+    >>> parse_prompt_attention('(unbalanced')
+    [['unbalanced', 1.1]]
+    >>> parse_prompt_attention('\(literal\]')
+    [['(literal]', 1.0]]
+    >>> parse_prompt_attention('(unnecessary)(parens)')
+    [['unnecessaryparens', 1.1]]
+    >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')
+    [['a ', 1.0],
+     ['house', 1.5730000000000004],
+     [' ', 1.1],
+     ['on', 1.0],
+     [' a ', 1.1],
+     ['hill', 0.55],
+     [', sun, ', 1.1],
+     ['sky', 1.4641000000000006],
+     ['.', 1.1]]
+    """
+
+    res = []
+    round_brackets = []
+    square_brackets = []
+
+    round_bracket_multiplier = 1.1
+    square_bracket_multiplier = 1 / 1.1
+
+    def multiply_range(start_position, multiplier):
+        for p in range(start_position, len(res)):
+            res[p][1] *= multiplier
+
+    for m in re_attention.finditer(text):
+        text = m.group(0)
+        weight = m.group(1)
+
+        if text.startswith("\\"):
+            res.append([text[1:], 1.0])
+        elif text == "(":
+            round_brackets.append(len(res))
+        elif text == "[":
+            square_brackets.append(len(res))
+        elif weight is not None and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), float(weight))
+        elif text == ")" and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), round_bracket_multiplier)
+        elif text == "]" and len(square_brackets) > 0:
+            multiply_range(square_brackets.pop(), square_bracket_multiplier)
+        else:
+            res.append([text, 1.0])
+
+    for pos in round_brackets:
+        multiply_range(pos, round_bracket_multiplier)
+
+    for pos in square_brackets:
+        multiply_range(pos, square_bracket_multiplier)
+
+    if len(res) == 0:
+        res = [["", 1.0]]
+
+    # merge runs of identical weights
+    i = 0
+    while i + 1 < len(res):
+        if res[i][1] == res[i + 1][1]:
+            res[i][0] += res[i + 1][0]
+            res.pop(i + 1)
+        else:
+            i += 1
+
+    return res
+
+
+def get_prompts_with_weights(tokenizer, prompt: List[str], max_length: int):
+    r"""
+    Tokenize a list of prompts and return its tokens with weights of each token.
+
+    No padding, starting or ending token is included.
+    """
+    tokens = []
+    weights = []
+    truncated = False
+    for text in prompt:
+        texts_and_weights = parse_prompt_attention(text)
+        text_token = []
+        text_weight = []
+        for word, weight in texts_and_weights:
+            # tokenize and discard the starting and the ending token
+            token = tokenizer(word).input_ids[1:-1]
+            text_token += token
+            # copy the weight by length of token
+            text_weight += [weight] * len(token)
+            # stop if the text is too long (longer than truncation limit)
+            if len(text_token) > max_length:
+                truncated = True
+                break
+        # truncate
+        if len(text_token) > max_length:
+            truncated = True
+            text_token = text_token[:max_length]
+            text_weight = text_weight[:max_length]
+        tokens.append(text_token)
+        weights.append(text_weight)
+    if truncated:
+        print("Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples")
+    return tokens, weights
+
+
+def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, no_boseos_middle=True, chunk_length=77):
+    r"""
+    Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length.
+    """
+    max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)
+    weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length
+    for i in range(len(tokens)):
+        tokens[i] = [bos] + tokens[i] + [eos] * (max_length - 1 - len(tokens[i]))
+        if no_boseos_middle:
+            weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))
+        else:
+            w = []
+            if len(weights[i]) == 0:
+                w = [1.0] * weights_length
+            else:
+                for j in range(max_embeddings_multiples):
+                    w.append(1.0)  # weight for starting token in this chunk
+                    w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]
+                    w.append(1.0)  # weight for ending token in this chunk
+                w += [1.0] * (weights_length - len(w))
+            weights[i] = w[:]
+
+    return tokens, weights
+
+
+def get_unweighted_text_embeddings(
+    tokenizer,
+    text_encoder,
+    text_input: torch.Tensor,
+    chunk_length: int,
+    clip_skip: int,
+    eos: int,
+    pad: int,
+    no_boseos_middle: Optional[bool] = True,
+):
+    """
+    When the length of tokens is a multiple of the capacity of the text encoder,
+    it should be split into chunks and sent to the text encoder individually.
+    """
+    max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)
+    if max_embeddings_multiples > 1:
+        text_embeddings = []
+        for i in range(max_embeddings_multiples):
+            # extract the i-th chunk
+            text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()
+
+            # cover the head and the tail by the starting and the ending tokens
+            text_input_chunk[:, 0] = text_input[0, 0]
+            if pad == eos:  # v1
+                text_input_chunk[:, -1] = text_input[0, -1]
+            else:  # v2
+                for j in range(len(text_input_chunk)):
+                    if text_input_chunk[j, -1] != eos and text_input_chunk[j, -1] != pad:  # 最後に普通の文字がある
+                        text_input_chunk[j, -1] = eos
+                    if text_input_chunk[j, 1] == pad:  # BOSだけであとはPAD
+                        text_input_chunk[j, 1] = eos
+
+            if clip_skip is None or clip_skip == 1:
+                text_embedding = text_encoder(text_input_chunk)[0]
+            else:
+                enc_out = text_encoder(text_input_chunk, output_hidden_states=True, return_dict=True)
+                text_embedding = enc_out["hidden_states"][-clip_skip]
+                text_embedding = text_encoder.text_model.final_layer_norm(text_embedding)
+
+            if no_boseos_middle:
+                if i == 0:
+                    # discard the ending token
+                    text_embedding = text_embedding[:, :-1]
+                elif i == max_embeddings_multiples - 1:
+                    # discard the starting token
+                    text_embedding = text_embedding[:, 1:]
+                else:
+                    # discard both starting and ending tokens
+                    text_embedding = text_embedding[:, 1:-1]
+
+            text_embeddings.append(text_embedding)
+        text_embeddings = torch.concat(text_embeddings, axis=1)
+    else:
+        if clip_skip is None or clip_skip == 1:
+            text_embeddings = text_encoder(text_input)[0]
+        else:
+            enc_out = text_encoder(text_input, output_hidden_states=True, return_dict=True)
+            text_embeddings = enc_out["hidden_states"][-clip_skip]
+            text_embeddings = text_encoder.text_model.final_layer_norm(text_embeddings)
+    return text_embeddings
+
+
+def get_weighted_text_embeddings(
+    tokenizer,
+    text_encoder,
+    prompt: Union[str, List[str]],
+    device,
+    max_embeddings_multiples: Optional[int] = 3,
+    no_boseos_middle: Optional[bool] = False,
+    clip_skip=None,
+):
+    r"""
+    Prompts can be assigned with local weights using brackets. For example,
+    prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful',
+    and the embedding tokens corresponding to the words get multiplied by a constant, 1.1.
+
+    Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean.
+
+    Args:
+        prompt (`str` or `List[str]`):
+            The prompt or prompts to guide the image generation.
+        max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+            The max multiple length of prompt embeddings compared to the max output length of text encoder.
+        no_boseos_middle (`bool`, *optional*, defaults to `False`):
+            If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and
+            ending token in each of the chunk in the middle.
+        skip_parsing (`bool`, *optional*, defaults to `False`):
+            Skip the parsing of brackets.
+        skip_weighting (`bool`, *optional*, defaults to `False`):
+            Skip the weighting. When the parsing is skipped, it is forced True.
+    """
+    max_length = (tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+    if isinstance(prompt, str):
+        prompt = [prompt]
+
+    prompt_tokens, prompt_weights = get_prompts_with_weights(tokenizer, prompt, max_length - 2)
+
+    # round up the longest length of tokens to a multiple of (model_max_length - 2)
+    max_length = max([len(token) for token in prompt_tokens])
+
+    max_embeddings_multiples = min(
+        max_embeddings_multiples,
+        (max_length - 1) // (tokenizer.model_max_length - 2) + 1,
+    )
+    max_embeddings_multiples = max(1, max_embeddings_multiples)
+    max_length = (tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+
+    # pad the length of tokens and weights
+    bos = tokenizer.bos_token_id
+    eos = tokenizer.eos_token_id
+    pad = tokenizer.pad_token_id
+    prompt_tokens, prompt_weights = pad_tokens_and_weights(
+        prompt_tokens,
+        prompt_weights,
+        max_length,
+        bos,
+        eos,
+        no_boseos_middle=no_boseos_middle,
+        chunk_length=tokenizer.model_max_length,
+    )
+    prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device=device)
+
+    # get the embeddings
+    text_embeddings = get_unweighted_text_embeddings(
+        tokenizer,
+        text_encoder,
+        prompt_tokens,
+        tokenizer.model_max_length,
+        clip_skip,
+        eos,
+        pad,
+        no_boseos_middle=no_boseos_middle,
+    )
+    prompt_weights = torch.tensor(prompt_weights, dtype=text_embeddings.dtype, device=device)
+
+    # assign weights to the prompts and normalize in the sense of mean
+    previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+    text_embeddings = text_embeddings * prompt_weights.unsqueeze(-1)
+    current_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+    text_embeddings = text_embeddings * (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)
+
+    return text_embeddings
+
+
+# https://wandb.ai/johnowhitaker/multires_noise/reports/Multi-Resolution-Noise-for-Diffusion-Model-Training--VmlldzozNjYyOTU2
+def pyramid_noise_like(noise, device, iterations=6, discount=0.4):
+    b, c, w, h = noise.shape  # EDIT: w and h get over-written, rename for a different variant!
+    u = torch.nn.Upsample(size=(w, h), mode="bilinear").to(device)
+    for i in range(iterations):
+        r = random.random() * 2 + 2  # Rather than always going 2x,
+        wn, hn = max(1, int(w / (r**i))), max(1, int(h / (r**i)))
+        noise += u(torch.randn(b, c, wn, hn).to(device)) * discount**i
+        if wn == 1 or hn == 1:
+            break  # Lowest resolution is 1x1
+    return noise / noise.std()  # Scaled back to roughly unit variance
+
+
+# https://www.crosslabs.org//blog/diffusion-with-offset-noise
+def apply_noise_offset(latents, noise, noise_offset, adaptive_noise_scale):
+    if noise_offset is None:
+        return noise
+    if adaptive_noise_scale is not None:
+        # latent shape: (batch_size, channels, height, width)
+        # abs mean value for each channel
+        latent_mean = torch.abs(latents.mean(dim=(2, 3), keepdim=True))
+
+        # multiply adaptive noise scale to the mean value and add it to the noise offset
+        noise_offset = noise_offset + adaptive_noise_scale * latent_mean
+        noise_offset = torch.clamp(noise_offset, 0.0, None)  # in case of adaptive noise scale is negative
+
+    noise = noise + noise_offset * torch.randn((latents.shape[0], latents.shape[1], 1, 1), device=latents.device)
+    return noise
+
+
+"""
+##########################################
+# Perlin Noise
+def rand_perlin_2d(device, shape, res, fade=lambda t: 6 * t**5 - 15 * t**4 + 10 * t**3):
+    delta = (res[0] / shape[0], res[1] / shape[1])
+    d = (shape[0] // res[0], shape[1] // res[1])
+
+    grid = (
+        torch.stack(
+            torch.meshgrid(torch.arange(0, res[0], delta[0], device=device), torch.arange(0, res[1], delta[1], device=device)),
+            dim=-1,
+        )
+        % 1
+    )
+    angles = 2 * torch.pi * torch.rand(res[0] + 1, res[1] + 1, device=device)
+    gradients = torch.stack((torch.cos(angles), torch.sin(angles)), dim=-1)
+
+    tile_grads = (
+        lambda slice1, slice2: gradients[slice1[0] : slice1[1], slice2[0] : slice2[1]]
+        .repeat_interleave(d[0], 0)
+        .repeat_interleave(d[1], 1)
+    )
+    dot = lambda grad, shift: (
+        torch.stack((grid[: shape[0], : shape[1], 0] + shift[0], grid[: shape[0], : shape[1], 1] + shift[1]), dim=-1)
+        * grad[: shape[0], : shape[1]]
+    ).sum(dim=-1)
+
+    n00 = dot(tile_grads([0, -1], [0, -1]), [0, 0])
+    n10 = dot(tile_grads([1, None], [0, -1]), [-1, 0])
+    n01 = dot(tile_grads([0, -1], [1, None]), [0, -1])
+    n11 = dot(tile_grads([1, None], [1, None]), [-1, -1])
+    t = fade(grid[: shape[0], : shape[1]])
+    return 1.414 * torch.lerp(torch.lerp(n00, n10, t[..., 0]), torch.lerp(n01, n11, t[..., 0]), t[..., 1])
+
+
+def rand_perlin_2d_octaves(device, shape, res, octaves=1, persistence=0.5):
+    noise = torch.zeros(shape, device=device)
+    frequency = 1
+    amplitude = 1
+    for _ in range(octaves):
+        noise += amplitude * rand_perlin_2d(device, shape, (frequency * res[0], frequency * res[1]))
+        frequency *= 2
+        amplitude *= persistence
+    return noise
+
+
+def perlin_noise(noise, device, octaves):
+    _, c, w, h = noise.shape
+    perlin = lambda: rand_perlin_2d_octaves(device, (w, h), (4, 4), octaves)
+    noise_perlin = []
+    for _ in range(c):
+        noise_perlin.append(perlin())
+    noise_perlin = torch.stack(noise_perlin).unsqueeze(0)   # (1, c, w, h)
+    noise += noise_perlin # broadcast for each batch
+    return noise / noise.std()  # Scaled back to roughly unit variance
+"""
diff --git a/library/dataset_balancing_gui.py b/library/dataset_balancing_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..93697f00d0df4ef51ffa1b8ba3cb83ee386b54d6
--- /dev/null
+++ b/library/dataset_balancing_gui.py
@@ -0,0 +1,156 @@
+import os
+import re
+import gradio as gr
+from easygui import msgbox, boolbox
+from .common_gui import get_folder_path
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+# def select_folder():
+#     # Open a file dialog to select a directory
+#     folder = filedialog.askdirectory()
+
+#     # Update the GUI to display the selected folder
+#     selected_folder_label.config(text=folder)
+
+
+def dataset_balancing(concept_repeats, folder, insecure):
+
+    if not concept_repeats > 0:
+        # Display an error message if the total number of repeats is not a valid integer
+        msgbox('Please enter a valid integer for the total number of repeats.')
+        return
+
+    concept_repeats = int(concept_repeats)
+
+    # Check if folder exist
+    if folder == '' or not os.path.isdir(folder):
+        msgbox('Please enter a valid folder for balancing.')
+        return
+
+    pattern = re.compile(r'^\d+_.+$')
+
+    # Iterate over the subdirectories in the selected folder
+    for subdir in os.listdir(folder):
+        if pattern.match(subdir) or insecure:
+            # Calculate the number of repeats for the current subdirectory
+            # Get a list of all the files in the folder
+            files = os.listdir(os.path.join(folder, subdir))
+
+            # Filter the list to include only image files
+            image_files = [
+                f
+                for f in files
+                if f.endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp'))
+            ]
+
+            # Count the number of image files
+            images = len(image_files)
+
+            if images == 0:
+                log.info(
+                    f'No images of type .jpg, .jpeg, .png, .gif, .webp were found in {os.listdir(os.path.join(folder, subdir))}'
+                )
+
+            # Check if the subdirectory name starts with a number inside braces,
+            # indicating that the repeats value should be multiplied
+            match = re.match(r'^\{(\d+\.?\d*)\}', subdir)
+            if match:
+                # Multiply the repeats value by the number inside the braces
+                if not images == 0:
+                    repeats = max(
+                        1,
+                        round(
+                            concept_repeats / images * float(match.group(1))
+                        ),
+                    )
+                else:
+                    repeats = 0
+                subdir = subdir[match.end() :]
+            else:
+                if not images == 0:
+                    repeats = max(1, round(concept_repeats / images))
+                else:
+                    repeats = 0
+
+            # Check if the subdirectory name already has a number at the beginning
+            match = re.match(r'^\d+_', subdir)
+            if match:
+                # Replace the existing number with the new number
+                old_name = os.path.join(folder, subdir)
+                new_name = os.path.join(
+                    folder, f'{repeats}_{subdir[match.end():]}'
+                )
+            else:
+                # Add the new number at the beginning of the name
+                old_name = os.path.join(folder, subdir)
+                new_name = os.path.join(folder, f'{repeats}_{subdir}')
+
+            os.rename(old_name, new_name)
+        else:
+            log.info(
+                f'Skipping folder {subdir} because it does not match kohya_ss expected syntax...'
+            )
+
+    msgbox('Dataset balancing completed...')
+
+
+def warning(insecure):
+    if insecure:
+        if boolbox(
+            f'WARNING!!! You have asked to rename non kohya_ss <num>_<text> folders...\n\nAre you sure you want to do that?',
+            choices=('Yes, I like danger', 'No, get me out of here'),
+        ):
+            return True
+        else:
+            return False
+
+
+def gradio_dataset_balancing_tab(headless=False):
+    with gr.Tab('Dreambooth/LoRA Dataset balancing'):
+        gr.Markdown(
+            'This utility will ensure that each concept folder in the dataset folder is used equally during the training process of the dreambooth machine learning model, regardless of the number of images in each folder. It will do this by renaming the concept folders to indicate the number of times they should be repeated during training.'
+        )
+        gr.Markdown(
+            'WARNING! The use of this utility on the wrong folder can lead to unexpected folder renaming!!!'
+        )
+        with gr.Row():
+            select_dataset_folder_input = gr.Textbox(
+                label='Dataset folder',
+                placeholder='Folder containing the concepts folders to balance...',
+                interactive=True,
+            )
+
+            select_dataset_folder_button = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            select_dataset_folder_button.click(
+                get_folder_path,
+                outputs=select_dataset_folder_input,
+                show_progress=False,
+            )
+
+            total_repeats_number = gr.Number(
+                value=1000,
+                interactive=True,
+                label='Training steps per concept per epoch',
+            )
+        with gr.Accordion('Advanced options', open=False):
+            insecure = gr.Checkbox(
+                value=False,
+                label='DANGER!!! -- Insecure folder renaming -- DANGER!!!',
+            )
+            insecure.change(warning, inputs=insecure, outputs=insecure)
+        balance_button = gr.Button('Balance dataset')
+        balance_button.click(
+            dataset_balancing,
+            inputs=[
+                total_repeats_number,
+                select_dataset_folder_input,
+                insecure,
+            ],
+            show_progress=False,
+        )
diff --git a/library/dreambooth_folder_creation_gui.py b/library/dreambooth_folder_creation_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..0456d885cd014d7ecf5e0dff207f291ee135735a
--- /dev/null
+++ b/library/dreambooth_folder_creation_gui.py
@@ -0,0 +1,216 @@
+import gradio as gr
+from easygui import diropenbox, msgbox
+from .common_gui import get_folder_path
+import shutil
+import os
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+
+def copy_info_to_Folders_tab(training_folder):
+    img_folder = os.path.join(training_folder, 'img')
+    if os.path.exists(os.path.join(training_folder, 'reg')):
+        reg_folder = os.path.join(training_folder, 'reg')
+    else:
+        reg_folder = ''
+    model_folder = os.path.join(training_folder, 'model')
+    log_folder = os.path.join(training_folder, 'log')
+
+    return img_folder, reg_folder, model_folder, log_folder
+
+
+def dreambooth_folder_preparation(
+    util_training_images_dir_input,
+    util_training_images_repeat_input,
+    util_instance_prompt_input,
+    util_regularization_images_dir_input,
+    util_regularization_images_repeat_input,
+    util_class_prompt_input,
+    util_training_dir_output,
+):
+
+    # Check if the input variables are empty
+    if not len(util_training_dir_output):
+        log.info(
+            "Destination training directory is missing... can't perform the required task..."
+        )
+        return
+    else:
+        # Create the util_training_dir_output directory if it doesn't exist
+        os.makedirs(util_training_dir_output, exist_ok=True)
+
+    # Check for instance prompt
+    if util_instance_prompt_input == '':
+        msgbox('Instance prompt missing...')
+        return
+
+    # Check for class prompt
+    if util_class_prompt_input == '':
+        msgbox('Class prompt missing...')
+        return
+
+    # Create the training_dir path
+    if util_training_images_dir_input == '':
+        log.info(
+            "Training images directory is missing... can't perform the required task..."
+        )
+        return
+    else:
+        training_dir = os.path.join(
+            util_training_dir_output,
+            f'img/{int(util_training_images_repeat_input)}_{util_instance_prompt_input} {util_class_prompt_input}',
+        )
+
+        # Remove folders if they exist
+        if os.path.exists(training_dir):
+            log.info(f'Removing existing directory {training_dir}...')
+            shutil.rmtree(training_dir)
+
+        # Copy the training images to their respective directories
+        log.info(f'Copy {util_training_images_dir_input} to {training_dir}...')
+        shutil.copytree(util_training_images_dir_input, training_dir)
+
+    if not util_regularization_images_dir_input == '':
+        # Create the regularization_dir path
+        if not util_regularization_images_repeat_input > 0:
+            log.info('Repeats is missing... not copying regularisation images...')
+        else:
+            regularization_dir = os.path.join(
+                util_training_dir_output,
+                f'reg/{int(util_regularization_images_repeat_input)}_{util_class_prompt_input}',
+            )
+
+            # Remove folders if they exist
+            if os.path.exists(regularization_dir):
+                log.info(f'Removing existing directory {regularization_dir}...')
+                shutil.rmtree(regularization_dir)
+
+            # Copy the regularisation images to their respective directories
+            log.info(
+                f'Copy {util_regularization_images_dir_input} to {regularization_dir}...'
+            )
+            shutil.copytree(
+                util_regularization_images_dir_input, regularization_dir
+            )
+    else:
+        log.info(
+            'Regularization images directory is missing... not copying regularisation images...'
+        )
+
+    # create log and model folder
+    # Check if the log folder exists and create it if it doesn't
+    if not os.path.exists(os.path.join(util_training_dir_output, 'log')):
+        os.makedirs(os.path.join(util_training_dir_output, 'log'))
+
+    # Check if the model folder exists and create it if it doesn't
+    if not os.path.exists(os.path.join(util_training_dir_output, 'model')):
+        os.makedirs(os.path.join(util_training_dir_output, 'model'))
+
+    log.info(
+        f'Done creating kohya_ss training folder structure at {util_training_dir_output}...'
+    )
+
+
+def gradio_dreambooth_folder_creation_tab(
+    train_data_dir_input=gr.Textbox(),
+    reg_data_dir_input=gr.Textbox(),
+    output_dir_input=gr.Textbox(),
+    logging_dir_input=gr.Textbox(),
+    headless=False,
+):
+    with gr.Tab('Dreambooth/LoRA Folder preparation'):
+        gr.Markdown(
+            'This utility will create the necessary folder structure for the training images and optional regularization images needed for the kohys_ss Dreambooth/LoRA method to function correctly.'
+        )
+        with gr.Row():
+            util_instance_prompt_input = gr.Textbox(
+                label='Instance prompt',
+                placeholder='Eg: asd',
+                interactive=True,
+            )
+            util_class_prompt_input = gr.Textbox(
+                label='Class prompt',
+                placeholder='Eg: person',
+                interactive=True,
+            )
+        with gr.Row():
+            util_training_images_dir_input = gr.Textbox(
+                label='Training images',
+                placeholder='Directory containing the training images',
+                interactive=True,
+            )
+            button_util_training_images_dir_input = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            button_util_training_images_dir_input.click(
+                get_folder_path,
+                outputs=util_training_images_dir_input,
+                show_progress=False,
+            )
+            util_training_images_repeat_input = gr.Number(
+                label='Repeats',
+                value=40,
+                interactive=True,
+                elem_id='number_input',
+            )
+        with gr.Row():
+            util_regularization_images_dir_input = gr.Textbox(
+                label='Regularisation images',
+                placeholder='(Optional) Directory containing the regularisation images',
+                interactive=True,
+            )
+            button_util_regularization_images_dir_input = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            button_util_regularization_images_dir_input.click(
+                get_folder_path,
+                outputs=util_regularization_images_dir_input,
+                show_progress=False,
+            )
+            util_regularization_images_repeat_input = gr.Number(
+                label='Repeats',
+                value=1,
+                interactive=True,
+                elem_id='number_input',
+            )
+        with gr.Row():
+            util_training_dir_output = gr.Textbox(
+                label='Destination training directory',
+                placeholder='Directory where formatted training and regularisation folders will be placed',
+                interactive=True,
+            )
+            button_util_training_dir_output = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            button_util_training_dir_output.click(
+                get_folder_path, outputs=util_training_dir_output
+            )
+        button_prepare_training_data = gr.Button('Prepare training data')
+        button_prepare_training_data.click(
+            dreambooth_folder_preparation,
+            inputs=[
+                util_training_images_dir_input,
+                util_training_images_repeat_input,
+                util_instance_prompt_input,
+                util_regularization_images_dir_input,
+                util_regularization_images_repeat_input,
+                util_class_prompt_input,
+                util_training_dir_output,
+            ],
+            show_progress=False,
+        )
+        button_copy_info_to_Folders_tab = gr.Button('Copy info to Folders Tab')
+        button_copy_info_to_Folders_tab.click(
+            copy_info_to_Folders_tab,
+            inputs=[util_training_dir_output],
+            outputs=[
+                train_data_dir_input,
+                reg_data_dir_input,
+                output_dir_input,
+                logging_dir_input,
+            ],
+            show_progress=False,
+        )
diff --git a/library/extract_lora_from_dylora_gui.py b/library/extract_lora_from_dylora_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e84fb58bf8b6d8ec22d9483afd5d121b968c17c
--- /dev/null
+++ b/library/extract_lora_from_dylora_gui.py
@@ -0,0 +1,121 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import (
+    get_saveasfilename_path,
+    get_file_path,
+)
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def extract_dylora(
+    model,
+    save_to,
+    unit,
+):
+    # Check for caption_text_input
+    if model == '':
+        msgbox('Invalid DyLoRA model file')
+        return
+
+    # Check if source model exist
+    if not os.path.isfile(model):
+        msgbox('The provided DyLoRA model is not a file')
+        return
+
+    run_cmd = (
+        f'{PYTHON} "{os.path.join("networks","extract_lora_from_dylora.py")}"'
+    )
+    run_cmd += f' --save_to "{save_to}"'
+    run_cmd += f' --model "{model}"'
+    run_cmd += f' --unit {unit}'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+    log.info('Done extracting DyLoRA...')
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_extract_dylora_tab(headless=False):
+    with gr.Tab('Extract DyLoRA'):
+        gr.Markdown(
+            'This utility can extract a DyLoRA network from a finetuned model.'
+        )
+        lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False)
+        lora_ext_name = gr.Textbox(value='LoRA model types', visible=False)
+
+        with gr.Row():
+            model = gr.Textbox(
+                label='DyLoRA model',
+                placeholder='Path to the DyLoRA model to extract from',
+                interactive=True,
+            )
+            button_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_model_file.click(
+                get_file_path,
+                inputs=[model, lora_ext, lora_ext_name],
+                outputs=model,
+                show_progress=False,
+            )
+
+            save_to = gr.Textbox(
+                label='Save to',
+                placeholder='path where to save the extracted LoRA model...',
+                interactive=True,
+            )
+            button_save_to = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_save_to.click(
+                get_saveasfilename_path,
+                inputs=[save_to, lora_ext, lora_ext_name],
+                outputs=save_to,
+                show_progress=False,
+            )
+            unit = gr.Slider(
+                minimum=1,
+                maximum=256,
+                label='Network Dimension (Rank)',
+                value=1,
+                step=1,
+                interactive=True,
+            )
+
+        extract_button = gr.Button('Extract LoRA model')
+
+        extract_button.click(
+            extract_dylora,
+            inputs=[
+                model,
+                save_to,
+                unit,
+            ],
+            show_progress=False,
+        )
diff --git a/library/extract_lora_gui.py b/library/extract_lora_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..45ddb5f6ddf8a2e9e887dd9924e1a1f2f14f0822
--- /dev/null
+++ b/library/extract_lora_gui.py
@@ -0,0 +1,189 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import (
+    get_saveasfilename_path,
+    get_any_file_path,
+    get_file_path,
+)
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def extract_lora(
+    model_tuned,
+    model_org,
+    save_to,
+    save_precision,
+    dim,
+    v2,
+    conv_dim,
+    device,
+):
+    # Check for caption_text_input
+    if model_tuned == '':
+        msgbox('Invalid finetuned model file')
+        return
+
+    if model_org == '':
+        msgbox('Invalid base model file')
+        return
+
+    # Check if source model exist
+    if not os.path.isfile(model_tuned):
+        msgbox('The provided finetuned model is not a file')
+        return
+
+    if not os.path.isfile(model_org):
+        msgbox('The provided base model is not a file')
+        return
+
+    run_cmd = (
+        f'{PYTHON} "{os.path.join("networks","extract_lora_from_models.py")}"'
+    )
+    run_cmd += f' --save_precision {save_precision}'
+    run_cmd += f' --save_to "{save_to}"'
+    run_cmd += f' --model_org "{model_org}"'
+    run_cmd += f' --model_tuned "{model_tuned}"'
+    run_cmd += f' --dim {dim}'
+    run_cmd += f' --device {device}'
+    if conv_dim > 0:
+        run_cmd += f' --conv_dim {conv_dim}'
+    if v2:
+        run_cmd += f' --v2'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_extract_lora_tab(headless=False):
+    with gr.Tab('Extract LoRA'):
+        gr.Markdown(
+            'This utility can extract a LoRA network from a finetuned model.'
+        )
+        lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False)
+        lora_ext_name = gr.Textbox(value='LoRA model types', visible=False)
+        model_ext = gr.Textbox(value='*.ckpt *.safetensors', visible=False)
+        model_ext_name = gr.Textbox(value='Model types', visible=False)
+
+        with gr.Row():
+            model_tuned = gr.Textbox(
+                label='Finetuned model',
+                placeholder='Path to the finetuned model to extract',
+                interactive=True,
+            )
+            button_model_tuned_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_model_tuned_file.click(
+                get_file_path,
+                inputs=[model_tuned, model_ext, model_ext_name],
+                outputs=model_tuned,
+                show_progress=False,
+            )
+
+            model_org = gr.Textbox(
+                label='Stable Diffusion base model',
+                placeholder='Stable Diffusion original model: ckpt or safetensors file',
+                interactive=True,
+            )
+            button_model_org_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_model_org_file.click(
+                get_file_path,
+                inputs=[model_org, model_ext, model_ext_name],
+                outputs=model_org,
+                show_progress=False,
+            )
+        with gr.Row():
+            save_to = gr.Textbox(
+                label='Save to',
+                placeholder='path where to save the extracted LoRA model...',
+                interactive=True,
+            )
+            button_save_to = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_save_to.click(
+                get_saveasfilename_path,
+                inputs=[save_to, lora_ext, lora_ext_name],
+                outputs=save_to,
+                show_progress=False,
+            )
+            save_precision = gr.Dropdown(
+                label='Save precision',
+                choices=['fp16', 'bf16', 'float'],
+                value='float',
+                interactive=True,
+            )
+        with gr.Row():
+            dim = gr.Slider(
+                minimum=4,
+                maximum=1024,
+                label='Network Dimension (Rank)',
+                value=128,
+                step=1,
+                interactive=True,
+            )
+            conv_dim = gr.Slider(
+                minimum=0,
+                maximum=1024,
+                label='Conv Dimension (Rank)',
+                value=128,
+                step=1,
+                interactive=True,
+            )
+            v2 = gr.Checkbox(label='v2', value=False, interactive=True)
+            device = gr.Dropdown(
+                label='Device',
+                choices=[
+                    'cpu',
+                    'cuda',
+                ],
+                value='cuda',
+                interactive=True,
+            )
+
+        extract_button = gr.Button('Extract LoRA model')
+
+        extract_button.click(
+            extract_lora,
+            inputs=[
+                model_tuned,
+                model_org,
+                save_to,
+                save_precision,
+                dim,
+                v2,
+                conv_dim,
+                device,
+            ],
+            show_progress=False,
+        )
diff --git a/library/extract_lycoris_locon_gui.py b/library/extract_lycoris_locon_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..d3c19da6f1520a323cf6b6ab415823acf9bc6caa
--- /dev/null
+++ b/library/extract_lycoris_locon_gui.py
@@ -0,0 +1,330 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import (
+    get_saveasfilename_path,
+    get_any_file_path,
+    get_file_path,
+)
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def extract_lycoris_locon(
+    db_model,
+    base_model,
+    output_name,
+    device,
+    is_v2,
+    mode,
+    linear_dim,
+    conv_dim,
+    linear_threshold,
+    conv_threshold,
+    linear_ratio,
+    conv_ratio,
+    linear_quantile,
+    conv_quantile,
+    use_sparse_bias,
+    sparsity,
+    disable_cp,
+):
+    # Check for caption_text_input
+    if db_model == '':
+        msgbox('Invalid finetuned model file')
+        return
+
+    if base_model == '':
+        msgbox('Invalid base model file')
+        return
+
+    # Check if source model exist
+    if not os.path.isfile(db_model):
+        msgbox('The provided finetuned model is not a file')
+        return
+
+    if not os.path.isfile(base_model):
+        msgbox('The provided base model is not a file')
+        return
+
+    run_cmd = f'{PYTHON} "{os.path.join("tools","lycoris_locon_extract.py")}"'
+    if is_v2:
+        run_cmd += f' --is_v2'
+    run_cmd += f' --device {device}'
+    run_cmd += f' --mode {mode}'
+    run_cmd += f' --safetensors'
+    if mode == 'fixed':
+        run_cmd += f' --linear_dim {linear_dim}'
+        run_cmd += f' --conv_dim {conv_dim}'
+    if mode == 'threshold':
+        run_cmd += f' --linear_threshold {linear_threshold}'
+        run_cmd += f' --conv_threshold {conv_threshold}'
+    if mode == 'ratio':
+        run_cmd += f' --linear_ratio {linear_ratio}'
+        run_cmd += f' --conv_ratio {conv_ratio}'
+    if mode == 'quantile':
+        run_cmd += f' --linear_quantile {linear_quantile}'
+        run_cmd += f' --conv_quantile {conv_quantile}'
+    if use_sparse_bias:
+        run_cmd += f' --use_sparse_bias'
+    run_cmd += f' --sparsity {sparsity}'
+    if disable_cp:
+        run_cmd += f' --disable_cp'
+    run_cmd += f' "{base_model}"'
+    run_cmd += f' "{db_model}"'
+    run_cmd += f' "{output_name}"'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+
+###
+# Gradio UI
+###
+# def update_mode(mode):
+#     # 'fixed', 'threshold','ratio','quantile'
+#     if mode == 'fixed':
+#         return gr.Row.update(visible=True), gr.Row.update(visible=False), gr.Row.update(visible=False), gr.Row.update(visible=False)
+#     if mode == 'threshold':
+#         return gr.Row.update(visible=False), gr.Row.update(visible=True), gr.Row.update(visible=False), gr.Row.update(visible=False)
+#     if mode == 'ratio':
+#         return gr.Row.update(visible=False), gr.Row.update(visible=False), gr.Row.update(visible=True), gr.Row.update(visible=False)
+#     if mode == 'threshold':
+#         return gr.Row.update(visible=False), gr.Row.update(visible=False), gr.Row.update(visible=False), gr.Row.update(visible=True)
+
+
+def update_mode(mode):
+    # Create a list of possible mode values
+    modes = ['fixed', 'threshold', 'ratio', 'quantile']
+
+    # Initialize an empty list to store visibility updates
+    updates = []
+
+    # Iterate through the possible modes
+    for m in modes:
+        # Add a visibility update for each mode, setting it to True if the input mode matches the current mode in the loop
+        updates.append(gr.Row.update(visible=(mode == m)))
+
+    # Return the visibility updates as a tuple
+    return tuple(updates)
+
+
+def gradio_extract_lycoris_locon_tab(headless=False):
+    with gr.Tab('Extract LyCORIS LoCON'):
+        gr.Markdown(
+            'This utility can extract a LyCORIS LoCon network from a finetuned model.'
+        )
+        lora_ext = gr.Textbox(
+            value='*.safetensors', visible=False
+        )   # lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False)
+        lora_ext_name = gr.Textbox(value='LoRA model types', visible=False)
+        model_ext = gr.Textbox(value='*.safetensors *.ckpt', visible=False)
+        model_ext_name = gr.Textbox(value='Model types', visible=False)
+
+        with gr.Row():
+            db_model = gr.Textbox(
+                label='Finetuned model',
+                placeholder='Path to the finetuned model to extract',
+                interactive=True,
+            )
+            button_db_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_db_model_file.click(
+                get_file_path,
+                inputs=[db_model, model_ext, model_ext_name],
+                outputs=db_model,
+                show_progress=False,
+            )
+
+            base_model = gr.Textbox(
+                label='Stable Diffusion base model',
+                placeholder='Stable Diffusion original model: ckpt or safetensors file',
+                interactive=True,
+            )
+            button_base_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_base_model_file.click(
+                get_file_path,
+                inputs=[base_model, model_ext, model_ext_name],
+                outputs=base_model,
+                show_progress=False,
+            )
+        with gr.Row():
+            output_name = gr.Textbox(
+                label='Save to',
+                placeholder='path where to save the extracted LoRA model...',
+                interactive=True,
+            )
+            button_output_name = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_output_name.click(
+                get_saveasfilename_path,
+                inputs=[output_name, lora_ext, lora_ext_name],
+                outputs=output_name,
+                show_progress=False,
+            )
+            device = gr.Dropdown(
+                label='Device',
+                choices=[
+                    'cpu',
+                    'cuda',
+                ],
+                value='cuda',
+                interactive=True,
+            )
+            is_v2 = gr.Checkbox(label='is v2', value=False, interactive=True)
+        mode = gr.Dropdown(
+            label='Mode',
+            choices=['fixed', 'threshold', 'ratio', 'quantile'],
+            value='fixed',
+            interactive=True,
+        )
+        with gr.Row(visible=True) as fixed:
+            linear_dim = gr.Slider(
+                minimum=1,
+                maximum=1024,
+                label='Network Dimension',
+                value=1,
+                step=1,
+                interactive=True,
+            )
+            conv_dim = gr.Slider(
+                minimum=1,
+                maximum=1024,
+                label='Conv Dimension',
+                value=1,
+                step=1,
+                interactive=True,
+            )
+        with gr.Row(visible=False) as threshold:
+            linear_threshold = gr.Slider(
+                minimum=0,
+                maximum=1,
+                label='Linear threshold',
+                value=0.65,
+                step=0.01,
+                interactive=True,
+                info='The higher the value, the smaller the file. Recommended starting value: 0.65',
+            )
+            conv_threshold = gr.Slider(
+                minimum=0,
+                maximum=1,
+                label='Conv threshold',
+                value=0.65,
+                step=0.01,
+                interactive=True,
+                info='The higher the value, the smaller the file. Recommended starting value: 0.65',
+            )
+        with gr.Row(visible=False) as ratio:
+            linear_ratio = gr.Slider(
+                minimum=0,
+                maximum=1,
+                label='Linear ratio',
+                value=0.75,
+                step=0.01,
+                interactive=True,
+                info='The higher the value, the smaller the file. Recommended starting value: 0.75',
+            )
+            conv_ratio = gr.Slider(
+                minimum=0,
+                maximum=1,
+                label='Conv ratio',
+                value=0.75,
+                step=0.01,
+                interactive=True,
+                info='The higher the value, the smaller the file. Recommended starting value: 0.75',
+            )
+        with gr.Row(visible=False) as quantile:
+            linear_quantile = gr.Slider(
+                minimum=0,
+                maximum=1,
+                label='Linear quantile',
+                value=0.75,
+                step=0.01,
+                interactive=True,
+                info='The higher the value, the larger the file. Recommended starting value: 0.75',
+            )
+            conv_quantile = gr.Slider(
+                minimum=0,
+                maximum=1,
+                label='Conv quantile',
+                value=0.75,
+                step=0.01,
+                interactive=True,
+                info='The higher the value, the larger the file. Recommended starting value: 0.75',
+            )
+        with gr.Row():
+            use_sparse_bias = gr.Checkbox(
+                label='Use sparse biais', value=False, interactive=True
+            )
+            sparsity = gr.Slider(
+                minimum=0,
+                maximum=1,
+                label='Sparsity',
+                value=0.98,
+                step=0.01,
+                interactive=True,
+            )
+            disable_cp = gr.Checkbox(
+                label='Disable CP decomposition', value=False, interactive=True
+            )
+        mode.change(
+            update_mode,
+            inputs=[mode],
+            outputs=[
+                fixed,
+                threshold,
+                ratio,
+                quantile,
+            ],
+        )
+
+        extract_button = gr.Button('Extract LyCORIS LoCon')
+
+        extract_button.click(
+            extract_lycoris_locon,
+            inputs=[
+                db_model,
+                base_model,
+                output_name,
+                device,
+                is_v2,
+                mode,
+                linear_dim,
+                conv_dim,
+                linear_threshold,
+                conv_threshold,
+                linear_ratio,
+                conv_ratio,
+                linear_quantile,
+                conv_quantile,
+                use_sparse_bias,
+                sparsity,
+                disable_cp,
+            ],
+            show_progress=False,
+        )
diff --git a/library/git_caption_gui.py b/library/git_caption_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..83e54cfbb32a4fb72380e6c1c926c49030fccb82
--- /dev/null
+++ b/library/git_caption_gui.py
@@ -0,0 +1,142 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import get_folder_path, add_pre_postfix
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def caption_images(
+    train_data_dir,
+    caption_ext,
+    batch_size,
+    max_data_loader_n_workers,
+    max_length,
+    model_id,
+    prefix,
+    postfix,
+):
+    # Check for images_dir_input
+    if train_data_dir == '':
+        msgbox('Image folder is missing...')
+        return
+
+    if caption_ext == '':
+        msgbox('Please provide an extension for the caption files.')
+        return
+
+    log.info(f'GIT captioning files in {train_data_dir}...')
+    run_cmd = f'{PYTHON} finetune/make_captions_by_git.py'
+    if not model_id == '':
+        run_cmd += f' --model_id="{model_id}"'
+    run_cmd += f' --batch_size="{int(batch_size)}"'
+    run_cmd += (
+        f' --max_data_loader_n_workers="{int(max_data_loader_n_workers)}"'
+    )
+    run_cmd += f' --max_length="{int(max_length)}"'
+    if caption_ext != '':
+        run_cmd += f' --caption_extension="{caption_ext}"'
+    run_cmd += f' "{train_data_dir}"'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+    # Add prefix and postfix
+    add_pre_postfix(
+        folder=train_data_dir,
+        caption_file_ext=caption_ext,
+        prefix=prefix,
+        postfix=postfix,
+    )
+
+    log.info('...captioning done')
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_git_caption_gui_tab(headless=False):
+    with gr.Tab('GIT Captioning'):
+        gr.Markdown(
+            'This utility will use GIT to caption files for each images in a folder.'
+        )
+        with gr.Row():
+            train_data_dir = gr.Textbox(
+                label='Image folder to caption',
+                placeholder='Directory containing the images to caption',
+                interactive=True,
+            )
+            button_train_data_dir_input = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            button_train_data_dir_input.click(
+                get_folder_path,
+                outputs=train_data_dir,
+                show_progress=False,
+            )
+        with gr.Row():
+            caption_ext = gr.Textbox(
+                label='Caption file extension',
+                placeholder='Extention for caption file. eg: .caption, .txt',
+                value='.txt',
+                interactive=True,
+            )
+
+            prefix = gr.Textbox(
+                label='Prefix to add to BLIP caption',
+                placeholder='(Optional)',
+                interactive=True,
+            )
+
+            postfix = gr.Textbox(
+                label='Postfix to add to BLIP caption',
+                placeholder='(Optional)',
+                interactive=True,
+            )
+
+            batch_size = gr.Number(
+                value=1, label='Batch size', interactive=True
+            )
+
+        with gr.Row():
+            max_data_loader_n_workers = gr.Number(
+                value=2, label='Number of workers', interactive=True
+            )
+            max_length = gr.Number(
+                value=75, label='Max length', interactive=True
+            )
+            model_id = gr.Textbox(
+                label='Model',
+                placeholder='(Optional) model id for GIT in Hugging Face',
+                interactive=True,
+            )
+
+        caption_button = gr.Button('Caption images')
+
+        caption_button.click(
+            caption_images,
+            inputs=[
+                train_data_dir,
+                caption_ext,
+                batch_size,
+                max_data_loader_n_workers,
+                max_length,
+                model_id,
+                prefix,
+                postfix,
+            ],
+            show_progress=False,
+        )
diff --git a/library/group_images_gui.py b/library/group_images_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..33536e7caa473469c85c0fb49d11e3bbe8a22c50
--- /dev/null
+++ b/library/group_images_gui.py
@@ -0,0 +1,136 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+from .common_gui import get_folder_path
+import os
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+def group_images(
+    input_folder,
+    output_folder,
+    group_size,
+    include_subfolders,
+    do_not_copy_other_files,
+    generate_captions,
+    caption_ext
+):
+    if input_folder == '':
+        msgbox('Input folder is missing...')
+        return
+
+    if output_folder == '':
+        msgbox('Please provide an output folder.')
+        return
+
+    log.info(f'Grouping images in {input_folder}...')
+
+    run_cmd = f'{PYTHON} "{os.path.join("tools","group_images.py")}"'
+    run_cmd += f' "{input_folder}"'
+    run_cmd += f' "{output_folder}"'
+    run_cmd += f' {(group_size)}'
+    if include_subfolders:
+        run_cmd += f' --include_subfolders'
+    if do_not_copy_other_files:
+        run_cmd += f' --do_not_copy_other_files'
+    if generate_captions:
+        run_cmd += f' --caption'
+        if caption_ext:
+            run_cmd += f' --caption_ext={caption_ext}'
+
+    log.info(run_cmd)
+
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+    log.info('...grouping done')
+
+
+def gradio_group_images_gui_tab(headless=False):
+    with gr.Tab('Group Images'):
+        gr.Markdown('This utility will group images in a folder based on their aspect ratio.')
+        
+        with gr.Row():
+            input_folder = gr.Textbox(
+                label='Input folder',
+                placeholder='Directory containing the images to group',
+                interactive=True,
+            )
+            button_input_folder = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            button_input_folder.click(
+                get_folder_path,
+                outputs=input_folder,
+                show_progress=False,
+            )
+
+            output_folder = gr.Textbox(
+                label='Output folder',
+                placeholder='Directory where the grouped images will be stored',
+                interactive=True,
+            )
+            button_output_folder = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            button_output_folder.click(
+                get_folder_path,
+                outputs=output_folder,
+                show_progress=False,
+            )
+        with gr.Row():
+            group_size = gr.Slider(
+                label='Group size',
+                info='Number of images to group together',
+                value='4',
+                minimum=1, maximum=64, step=1,
+                interactive=True,
+            )
+
+            include_subfolders = gr.Checkbox(
+                label='Include Subfolders',
+                value=False,
+                info='Include images in subfolders as well',
+            )
+
+            do_not_copy_other_files = gr.Checkbox(
+                label='Do not copy other files',
+                value=False,
+                info='Do not copy other files in the input folder to the output folder',
+            )
+            
+            generate_captions = gr.Checkbox(
+                label='Generate Captions',
+                value=False,
+                info='Generate caption files for the grouped images based on their folder name',
+            )
+            
+            caption_ext = gr.Textbox(
+                label='Caption Extension',
+                placeholder='Caption file extension (e.g., .txt)',
+                value='.txt',
+                interactive=True,
+            )
+
+        group_images_button = gr.Button('Group images')
+
+        group_images_button.click(
+            group_images,
+            inputs=[
+                input_folder,
+                output_folder,
+                group_size,
+                include_subfolders,
+                do_not_copy_other_files,
+                generate_captions,
+                caption_ext,
+            ],
+            show_progress=False,
+        )
diff --git a/library/huggingface_util.py b/library/huggingface_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..1dc496ff5c815368c98b6d14c27c1a689d36890f
--- /dev/null
+++ b/library/huggingface_util.py
@@ -0,0 +1,81 @@
+from typing import Union, BinaryIO
+from huggingface_hub import HfApi
+from pathlib import Path
+import argparse
+import os
+from library.utils import fire_in_thread
+
+
+def exists_repo(repo_id: str, repo_type: str, revision: str = "main", token: str = None):
+    api = HfApi(
+        token=token,
+    )
+    try:
+        api.repo_info(repo_id=repo_id, revision=revision, repo_type=repo_type)
+        return True
+    except:
+        return False
+
+
+def upload(
+    args: argparse.Namespace,
+    src: Union[str, Path, bytes, BinaryIO],
+    dest_suffix: str = "",
+    force_sync_upload: bool = False,
+):
+    repo_id = args.huggingface_repo_id
+    repo_type = args.huggingface_repo_type
+    token = args.huggingface_token
+    path_in_repo = args.huggingface_path_in_repo + dest_suffix
+    private = args.huggingface_repo_visibility is None or args.huggingface_repo_visibility != "public"
+    api = HfApi(token=token)
+    if not exists_repo(repo_id=repo_id, repo_type=repo_type, token=token):
+        try:
+            api.create_repo(repo_id=repo_id, repo_type=repo_type, private=private)
+        except Exception as e:  # とりあえずRepositoryNotFoundErrorは確認したが他にあると困るので
+            print("===========================================")
+            print(f"failed to create HuggingFace repo / HuggingFaceのリポジトリの作成に失敗しました : {e}")
+            print("===========================================")
+
+    is_folder = (type(src) == str and os.path.isdir(src)) or (isinstance(src, Path) and src.is_dir())
+
+    def uploader():
+        try:
+            if is_folder:
+                api.upload_folder(
+                    repo_id=repo_id,
+                    repo_type=repo_type,
+                    folder_path=src,
+                    path_in_repo=path_in_repo,
+                )
+            else:
+                api.upload_file(
+                    repo_id=repo_id,
+                    repo_type=repo_type,
+                    path_or_fileobj=src,
+                    path_in_repo=path_in_repo,
+                )
+        except Exception as e:  # RuntimeErrorを確認済みだが他にあると困るので
+            print("===========================================")
+            print(f"failed to upload to HuggingFace / HuggingFaceへのアップロードに失敗しました : {e}")
+            print("===========================================")
+
+    if args.async_upload and not force_sync_upload:
+        fire_in_thread(uploader)
+    else:
+        uploader()
+
+
+def list_dir(
+    repo_id: str,
+    subfolder: str,
+    repo_type: str,
+    revision: str = "main",
+    token: str = None,
+):
+    api = HfApi(
+        token=token,
+    )
+    repo_info = api.repo_info(repo_id=repo_id, revision=revision, repo_type=repo_type)
+    file_list = [file for file in repo_info.siblings if file.rfilename.startswith(subfolder)]
+    return file_list
diff --git a/library/hypernetwork.py b/library/hypernetwork.py
new file mode 100644
index 0000000000000000000000000000000000000000..fbd3fb24e1a5bc314b282407d1c6282a197d96a3
--- /dev/null
+++ b/library/hypernetwork.py
@@ -0,0 +1,223 @@
+import torch
+import torch.nn.functional as F
+from diffusers.models.attention_processor import (
+    Attention,
+    AttnProcessor2_0,
+    SlicedAttnProcessor,
+    XFormersAttnProcessor
+)
+
+try:
+    import xformers.ops
+except:
+    xformers = None
+
+
+loaded_networks = []
+
+
+def apply_single_hypernetwork(
+    hypernetwork, hidden_states, encoder_hidden_states
+):
+    context_k, context_v = hypernetwork.forward(hidden_states, encoder_hidden_states)
+    return context_k, context_v
+
+
+def apply_hypernetworks(context_k, context_v, layer=None):
+    if len(loaded_networks) == 0:
+        return context_v, context_v
+    for hypernetwork in loaded_networks:
+        context_k, context_v = hypernetwork.forward(context_k, context_v)
+
+    context_k = context_k.to(dtype=context_k.dtype)
+    context_v = context_v.to(dtype=context_k.dtype)
+
+    return context_k, context_v
+
+
+
+def xformers_forward(
+    self: XFormersAttnProcessor,
+    attn: Attention,
+    hidden_states: torch.Tensor,
+    encoder_hidden_states: torch.Tensor = None,
+    attention_mask: torch.Tensor = None,
+):
+    batch_size, sequence_length, _ = (
+        hidden_states.shape
+        if encoder_hidden_states is None
+        else encoder_hidden_states.shape
+    )
+
+    attention_mask = attn.prepare_attention_mask(
+        attention_mask, sequence_length, batch_size
+    )
+
+    query = attn.to_q(hidden_states)
+
+    if encoder_hidden_states is None:
+        encoder_hidden_states = hidden_states
+    elif attn.norm_cross:
+        encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+    context_k, context_v = apply_hypernetworks(hidden_states, encoder_hidden_states)
+
+    key = attn.to_k(context_k)
+    value = attn.to_v(context_v)
+
+    query = attn.head_to_batch_dim(query).contiguous()
+    key = attn.head_to_batch_dim(key).contiguous()
+    value = attn.head_to_batch_dim(value).contiguous()
+
+    hidden_states = xformers.ops.memory_efficient_attention(
+        query,
+        key,
+        value,
+        attn_bias=attention_mask,
+        op=self.attention_op,
+        scale=attn.scale,
+    )
+    hidden_states = hidden_states.to(query.dtype)
+    hidden_states = attn.batch_to_head_dim(hidden_states)
+
+    # linear proj
+    hidden_states = attn.to_out[0](hidden_states)
+    # dropout
+    hidden_states = attn.to_out[1](hidden_states)
+    return hidden_states
+
+
+def sliced_attn_forward(
+    self: SlicedAttnProcessor,
+    attn: Attention,
+    hidden_states: torch.Tensor,
+    encoder_hidden_states: torch.Tensor = None,
+    attention_mask: torch.Tensor = None,
+):
+    batch_size, sequence_length, _ = (
+        hidden_states.shape
+        if encoder_hidden_states is None
+        else encoder_hidden_states.shape
+    )
+    attention_mask = attn.prepare_attention_mask(
+        attention_mask, sequence_length, batch_size
+    )
+
+    query = attn.to_q(hidden_states)
+    dim = query.shape[-1]
+    query = attn.head_to_batch_dim(query)
+
+    if encoder_hidden_states is None:
+        encoder_hidden_states = hidden_states
+    elif attn.norm_cross:
+        encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+    context_k, context_v = apply_hypernetworks(hidden_states, encoder_hidden_states)
+
+    key = attn.to_k(context_k)
+    value = attn.to_v(context_v)
+    key = attn.head_to_batch_dim(key)
+    value = attn.head_to_batch_dim(value)
+
+    batch_size_attention, query_tokens, _ = query.shape
+    hidden_states = torch.zeros(
+        (batch_size_attention, query_tokens, dim // attn.heads),
+        device=query.device,
+        dtype=query.dtype,
+    )
+
+    for i in range(batch_size_attention // self.slice_size):
+        start_idx = i * self.slice_size
+        end_idx = (i + 1) * self.slice_size
+
+        query_slice = query[start_idx:end_idx]
+        key_slice = key[start_idx:end_idx]
+        attn_mask_slice = (
+            attention_mask[start_idx:end_idx] if attention_mask is not None else None
+        )
+
+        attn_slice = attn.get_attention_scores(query_slice, key_slice, attn_mask_slice)
+
+        attn_slice = torch.bmm(attn_slice, value[start_idx:end_idx])
+
+        hidden_states[start_idx:end_idx] = attn_slice
+
+    hidden_states = attn.batch_to_head_dim(hidden_states)
+
+    # linear proj
+    hidden_states = attn.to_out[0](hidden_states)
+    # dropout
+    hidden_states = attn.to_out[1](hidden_states)
+
+    return hidden_states
+
+
+def v2_0_forward(
+    self: AttnProcessor2_0,
+    attn: Attention,
+    hidden_states,
+    encoder_hidden_states=None,
+    attention_mask=None,
+):
+    batch_size, sequence_length, _ = (
+        hidden_states.shape
+        if encoder_hidden_states is None
+        else encoder_hidden_states.shape
+    )
+    inner_dim = hidden_states.shape[-1]
+
+    if attention_mask is not None:
+        attention_mask = attn.prepare_attention_mask(
+            attention_mask, sequence_length, batch_size
+        )
+        # scaled_dot_product_attention expects attention_mask shape to be
+        # (batch, heads, source_length, target_length)
+        attention_mask = attention_mask.view(
+            batch_size, attn.heads, -1, attention_mask.shape[-1]
+        )
+
+    query = attn.to_q(hidden_states)
+
+    if encoder_hidden_states is None:
+        encoder_hidden_states = hidden_states
+    elif attn.norm_cross:
+        encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states)
+
+    context_k, context_v = apply_hypernetworks(hidden_states, encoder_hidden_states)
+
+    key = attn.to_k(context_k)
+    value = attn.to_v(context_v)
+
+    head_dim = inner_dim // attn.heads
+    query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+    key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+    value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+    # the output of sdp = (batch, num_heads, seq_len, head_dim)
+    # TODO: add support for attn.scale when we move to Torch 2.1
+    hidden_states = F.scaled_dot_product_attention(
+        query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+    )
+
+    hidden_states = hidden_states.transpose(1, 2).reshape(
+        batch_size, -1, attn.heads * head_dim
+    )
+    hidden_states = hidden_states.to(query.dtype)
+
+    # linear proj
+    hidden_states = attn.to_out[0](hidden_states)
+    # dropout
+    hidden_states = attn.to_out[1](hidden_states)
+    return hidden_states
+
+
+def replace_attentions_for_hypernetwork():
+    import diffusers.models.attention_processor
+
+    diffusers.models.attention_processor.XFormersAttnProcessor.__call__ = (
+        xformers_forward
+    )
+    diffusers.models.attention_processor.SlicedAttnProcessor.__call__ = (
+        sliced_attn_forward
+    )
+    diffusers.models.attention_processor.AttnProcessor2_0.__call__ = v2_0_forward
diff --git a/library/lpw_stable_diffusion.py b/library/lpw_stable_diffusion.py
new file mode 100644
index 0000000000000000000000000000000000000000..2605c86426e6932b0fb6429f6276fecc203e8f9c
--- /dev/null
+++ b/library/lpw_stable_diffusion.py
@@ -0,0 +1,1261 @@
+# copy from https://github.com/huggingface/diffusers/blob/main/examples/community/lpw_stable_diffusion.py
+# and modify to support SD2.x
+
+import inspect
+import re
+from typing import Callable, List, Optional, Union
+
+import numpy as np
+import PIL.Image
+import torch
+from packaging import version
+from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer
+
+import diffusers
+from diffusers import SchedulerMixin, StableDiffusionPipeline
+from diffusers.models import AutoencoderKL, UNet2DConditionModel
+from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker
+from diffusers.utils import logging
+
+
+try:
+    from diffusers.utils import PIL_INTERPOLATION
+except ImportError:
+    if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"):
+        PIL_INTERPOLATION = {
+            "linear": PIL.Image.Resampling.BILINEAR,
+            "bilinear": PIL.Image.Resampling.BILINEAR,
+            "bicubic": PIL.Image.Resampling.BICUBIC,
+            "lanczos": PIL.Image.Resampling.LANCZOS,
+            "nearest": PIL.Image.Resampling.NEAREST,
+        }
+    else:
+        PIL_INTERPOLATION = {
+            "linear": PIL.Image.LINEAR,
+            "bilinear": PIL.Image.BILINEAR,
+            "bicubic": PIL.Image.BICUBIC,
+            "lanczos": PIL.Image.LANCZOS,
+            "nearest": PIL.Image.NEAREST,
+        }
+# ------------------------------------------------------------------------------
+
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+re_attention = re.compile(
+    r"""
+\\\(|
+\\\)|
+\\\[|
+\\]|
+\\\\|
+\\|
+\(|
+\[|
+:([+-]?[.\d]+)\)|
+\)|
+]|
+[^\\()\[\]:]+|
+:
+""",
+    re.X,
+)
+
+
+def parse_prompt_attention(text):
+    """
+    Parses a string with attention tokens and returns a list of pairs: text and its associated weight.
+    Accepted tokens are:
+      (abc) - increases attention to abc by a multiplier of 1.1
+      (abc:3.12) - increases attention to abc by a multiplier of 3.12
+      [abc] - decreases attention to abc by a multiplier of 1.1
+      \( - literal character '('
+      \[ - literal character '['
+      \) - literal character ')'
+      \] - literal character ']'
+      \\ - literal character '\'
+      anything else - just text
+    >>> parse_prompt_attention('normal text')
+    [['normal text', 1.0]]
+    >>> parse_prompt_attention('an (important) word')
+    [['an ', 1.0], ['important', 1.1], [' word', 1.0]]
+    >>> parse_prompt_attention('(unbalanced')
+    [['unbalanced', 1.1]]
+    >>> parse_prompt_attention('\(literal\]')
+    [['(literal]', 1.0]]
+    >>> parse_prompt_attention('(unnecessary)(parens)')
+    [['unnecessaryparens', 1.1]]
+    >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')
+    [['a ', 1.0],
+     ['house', 1.5730000000000004],
+     [' ', 1.1],
+     ['on', 1.0],
+     [' a ', 1.1],
+     ['hill', 0.55],
+     [', sun, ', 1.1],
+     ['sky', 1.4641000000000006],
+     ['.', 1.1]]
+    """
+
+    res = []
+    round_brackets = []
+    square_brackets = []
+
+    round_bracket_multiplier = 1.1
+    square_bracket_multiplier = 1 / 1.1
+
+    def multiply_range(start_position, multiplier):
+        for p in range(start_position, len(res)):
+            res[p][1] *= multiplier
+
+    for m in re_attention.finditer(text):
+        text = m.group(0)
+        weight = m.group(1)
+
+        if text.startswith("\\"):
+            res.append([text[1:], 1.0])
+        elif text == "(":
+            round_brackets.append(len(res))
+        elif text == "[":
+            square_brackets.append(len(res))
+        elif weight is not None and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), float(weight))
+        elif text == ")" and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), round_bracket_multiplier)
+        elif text == "]" and len(square_brackets) > 0:
+            multiply_range(square_brackets.pop(), square_bracket_multiplier)
+        else:
+            res.append([text, 1.0])
+
+    for pos in round_brackets:
+        multiply_range(pos, round_bracket_multiplier)
+
+    for pos in square_brackets:
+        multiply_range(pos, square_bracket_multiplier)
+
+    if len(res) == 0:
+        res = [["", 1.0]]
+
+    # merge runs of identical weights
+    i = 0
+    while i + 1 < len(res):
+        if res[i][1] == res[i + 1][1]:
+            res[i][0] += res[i + 1][0]
+            res.pop(i + 1)
+        else:
+            i += 1
+
+    return res
+
+
+def get_prompts_with_weights(pipe: StableDiffusionPipeline, prompt: List[str], max_length: int):
+    r"""
+    Tokenize a list of prompts and return its tokens with weights of each token.
+
+    No padding, starting or ending token is included.
+    """
+    tokens = []
+    weights = []
+    truncated = False
+    for text in prompt:
+        texts_and_weights = parse_prompt_attention(text)
+        text_token = []
+        text_weight = []
+        for word, weight in texts_and_weights:
+            # tokenize and discard the starting and the ending token
+            token = pipe.tokenizer(word).input_ids[1:-1]
+            text_token += token
+            # copy the weight by length of token
+            text_weight += [weight] * len(token)
+            # stop if the text is too long (longer than truncation limit)
+            if len(text_token) > max_length:
+                truncated = True
+                break
+        # truncate
+        if len(text_token) > max_length:
+            truncated = True
+            text_token = text_token[:max_length]
+            text_weight = text_weight[:max_length]
+        tokens.append(text_token)
+        weights.append(text_weight)
+    if truncated:
+        logger.warning("Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples")
+    return tokens, weights
+
+
+def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, no_boseos_middle=True, chunk_length=77):
+    r"""
+    Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length.
+    """
+    max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)
+    weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length
+    for i in range(len(tokens)):
+        tokens[i] = [bos] + tokens[i] + [eos] * (max_length - 1 - len(tokens[i]))
+        if no_boseos_middle:
+            weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))
+        else:
+            w = []
+            if len(weights[i]) == 0:
+                w = [1.0] * weights_length
+            else:
+                for j in range(max_embeddings_multiples):
+                    w.append(1.0)  # weight for starting token in this chunk
+                    w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]
+                    w.append(1.0)  # weight for ending token in this chunk
+                w += [1.0] * (weights_length - len(w))
+            weights[i] = w[:]
+
+    return tokens, weights
+
+
+def get_unweighted_text_embeddings(
+    pipe: StableDiffusionPipeline,
+    text_input: torch.Tensor,
+    chunk_length: int,
+    clip_skip: int,
+    eos: int,
+    pad: int,
+    no_boseos_middle: Optional[bool] = True,
+):
+    """
+    When the length of tokens is a multiple of the capacity of the text encoder,
+    it should be split into chunks and sent to the text encoder individually.
+    """
+    max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)
+    if max_embeddings_multiples > 1:
+        text_embeddings = []
+        for i in range(max_embeddings_multiples):
+            # extract the i-th chunk
+            text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()
+
+            # cover the head and the tail by the starting and the ending tokens
+            text_input_chunk[:, 0] = text_input[0, 0]
+            if pad == eos:  # v1
+                text_input_chunk[:, -1] = text_input[0, -1]
+            else:  # v2
+                for j in range(len(text_input_chunk)):
+                    if text_input_chunk[j, -1] != eos and text_input_chunk[j, -1] != pad:  # 最後に普通の文字がある
+                        text_input_chunk[j, -1] = eos
+                    if text_input_chunk[j, 1] == pad:  # BOSだけであとはPAD
+                        text_input_chunk[j, 1] = eos
+
+            if clip_skip is None or clip_skip == 1:
+                text_embedding = pipe.text_encoder(text_input_chunk)[0]
+            else:
+                enc_out = pipe.text_encoder(text_input_chunk, output_hidden_states=True, return_dict=True)
+                text_embedding = enc_out["hidden_states"][-clip_skip]
+                text_embedding = pipe.text_encoder.text_model.final_layer_norm(text_embedding)
+
+            if no_boseos_middle:
+                if i == 0:
+                    # discard the ending token
+                    text_embedding = text_embedding[:, :-1]
+                elif i == max_embeddings_multiples - 1:
+                    # discard the starting token
+                    text_embedding = text_embedding[:, 1:]
+                else:
+                    # discard both starting and ending tokens
+                    text_embedding = text_embedding[:, 1:-1]
+
+            text_embeddings.append(text_embedding)
+        text_embeddings = torch.concat(text_embeddings, axis=1)
+    else:
+        if clip_skip is None or clip_skip == 1:
+            text_embeddings = pipe.text_encoder(text_input)[0]
+        else:
+            enc_out = pipe.text_encoder(text_input, output_hidden_states=True, return_dict=True)
+            text_embeddings = enc_out["hidden_states"][-clip_skip]
+            text_embeddings = pipe.text_encoder.text_model.final_layer_norm(text_embeddings)
+    return text_embeddings
+
+
+def get_weighted_text_embeddings(
+    pipe: StableDiffusionPipeline,
+    prompt: Union[str, List[str]],
+    uncond_prompt: Optional[Union[str, List[str]]] = None,
+    max_embeddings_multiples: Optional[int] = 3,
+    no_boseos_middle: Optional[bool] = False,
+    skip_parsing: Optional[bool] = False,
+    skip_weighting: Optional[bool] = False,
+    clip_skip=None,
+):
+    r"""
+    Prompts can be assigned with local weights using brackets. For example,
+    prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful',
+    and the embedding tokens corresponding to the words get multiplied by a constant, 1.1.
+
+    Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean.
+
+    Args:
+        pipe (`StableDiffusionPipeline`):
+            Pipe to provide access to the tokenizer and the text encoder.
+        prompt (`str` or `List[str]`):
+            The prompt or prompts to guide the image generation.
+        uncond_prompt (`str` or `List[str]`):
+            The unconditional prompt or prompts for guide the image generation. If unconditional prompt
+            is provided, the embeddings of prompt and uncond_prompt are concatenated.
+        max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+            The max multiple length of prompt embeddings compared to the max output length of text encoder.
+        no_boseos_middle (`bool`, *optional*, defaults to `False`):
+            If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and
+            ending token in each of the chunk in the middle.
+        skip_parsing (`bool`, *optional*, defaults to `False`):
+            Skip the parsing of brackets.
+        skip_weighting (`bool`, *optional*, defaults to `False`):
+            Skip the weighting. When the parsing is skipped, it is forced True.
+    """
+    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+    if isinstance(prompt, str):
+        prompt = [prompt]
+
+    if not skip_parsing:
+        prompt_tokens, prompt_weights = get_prompts_with_weights(pipe, prompt, max_length - 2)
+        if uncond_prompt is not None:
+            if isinstance(uncond_prompt, str):
+                uncond_prompt = [uncond_prompt]
+            uncond_tokens, uncond_weights = get_prompts_with_weights(pipe, uncond_prompt, max_length - 2)
+    else:
+        prompt_tokens = [token[1:-1] for token in pipe.tokenizer(prompt, max_length=max_length, truncation=True).input_ids]
+        prompt_weights = [[1.0] * len(token) for token in prompt_tokens]
+        if uncond_prompt is not None:
+            if isinstance(uncond_prompt, str):
+                uncond_prompt = [uncond_prompt]
+            uncond_tokens = [
+                token[1:-1] for token in pipe.tokenizer(uncond_prompt, max_length=max_length, truncation=True).input_ids
+            ]
+            uncond_weights = [[1.0] * len(token) for token in uncond_tokens]
+
+    # round up the longest length of tokens to a multiple of (model_max_length - 2)
+    max_length = max([len(token) for token in prompt_tokens])
+    if uncond_prompt is not None:
+        max_length = max(max_length, max([len(token) for token in uncond_tokens]))
+
+    max_embeddings_multiples = min(
+        max_embeddings_multiples,
+        (max_length - 1) // (pipe.tokenizer.model_max_length - 2) + 1,
+    )
+    max_embeddings_multiples = max(1, max_embeddings_multiples)
+    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+
+    # pad the length of tokens and weights
+    bos = pipe.tokenizer.bos_token_id
+    eos = pipe.tokenizer.eos_token_id
+    pad = pipe.tokenizer.pad_token_id
+    prompt_tokens, prompt_weights = pad_tokens_and_weights(
+        prompt_tokens,
+        prompt_weights,
+        max_length,
+        bos,
+        eos,
+        no_boseos_middle=no_boseos_middle,
+        chunk_length=pipe.tokenizer.model_max_length,
+    )
+    prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device=pipe.device)
+    if uncond_prompt is not None:
+        uncond_tokens, uncond_weights = pad_tokens_and_weights(
+            uncond_tokens,
+            uncond_weights,
+            max_length,
+            bos,
+            eos,
+            no_boseos_middle=no_boseos_middle,
+            chunk_length=pipe.tokenizer.model_max_length,
+        )
+        uncond_tokens = torch.tensor(uncond_tokens, dtype=torch.long, device=pipe.device)
+
+    # get the embeddings
+    text_embeddings = get_unweighted_text_embeddings(
+        pipe,
+        prompt_tokens,
+        pipe.tokenizer.model_max_length,
+        clip_skip,
+        eos,
+        pad,
+        no_boseos_middle=no_boseos_middle,
+    )
+    prompt_weights = torch.tensor(prompt_weights, dtype=text_embeddings.dtype, device=pipe.device)
+    if uncond_prompt is not None:
+        uncond_embeddings = get_unweighted_text_embeddings(
+            pipe,
+            uncond_tokens,
+            pipe.tokenizer.model_max_length,
+            clip_skip,
+            eos,
+            pad,
+            no_boseos_middle=no_boseos_middle,
+        )
+        uncond_weights = torch.tensor(uncond_weights, dtype=uncond_embeddings.dtype, device=pipe.device)
+
+    # assign weights to the prompts and normalize in the sense of mean
+    # TODO: should we normalize by chunk or in a whole (current implementation)?
+    if (not skip_parsing) and (not skip_weighting):
+        previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+        text_embeddings *= prompt_weights.unsqueeze(-1)
+        current_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+        text_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)
+        if uncond_prompt is not None:
+            previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)
+            uncond_embeddings *= uncond_weights.unsqueeze(-1)
+            current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)
+            uncond_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)
+
+    if uncond_prompt is not None:
+        return text_embeddings, uncond_embeddings
+    return text_embeddings, None
+
+
+def preprocess_image(image):
+    w, h = image.size
+    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32
+    image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"])
+    image = np.array(image).astype(np.float32) / 255.0
+    image = image[None].transpose(0, 3, 1, 2)
+    image = torch.from_numpy(image)
+    return 2.0 * image - 1.0
+
+
+def preprocess_mask(mask, scale_factor=8):
+    mask = mask.convert("L")
+    w, h = mask.size
+    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32
+    mask = mask.resize((w // scale_factor, h // scale_factor), resample=PIL_INTERPOLATION["nearest"])
+    mask = np.array(mask).astype(np.float32) / 255.0
+    mask = np.tile(mask, (4, 1, 1))
+    mask = mask[None].transpose(0, 1, 2, 3)  # what does this step do?
+    mask = 1 - mask  # repaint white, keep black
+    mask = torch.from_numpy(mask)
+    return mask
+
+
+def prepare_controlnet_image(
+    image: PIL.Image.Image,
+    width: int,
+    height: int,
+    batch_size: int,
+    num_images_per_prompt: int,
+    device: torch.device,
+    dtype: torch.dtype,
+    do_classifier_free_guidance: bool = False,
+    guess_mode: bool = False,
+):
+    if not isinstance(image, torch.Tensor):
+        if isinstance(image, PIL.Image.Image):
+            image = [image]
+
+        if isinstance(image[0], PIL.Image.Image):
+            images = []
+
+            for image_ in image:
+                image_ = image_.convert("RGB")
+                image_ = image_.resize(
+                    (width, height), resample=PIL_INTERPOLATION["lanczos"]
+                )
+                image_ = np.array(image_)
+                image_ = image_[None, :]
+                images.append(image_)
+
+            image = images
+
+            image = np.concatenate(image, axis=0)
+            image = np.array(image).astype(np.float32) / 255.0
+            image = image.transpose(0, 3, 1, 2)
+            image = torch.from_numpy(image)
+        elif isinstance(image[0], torch.Tensor):
+            image = torch.cat(image, dim=0)
+
+    image_batch_size = image.shape[0]
+
+    if image_batch_size == 1:
+        repeat_by = batch_size
+    else:
+        # image batch size is the same as prompt batch size
+        repeat_by = num_images_per_prompt
+
+    image = image.repeat_interleave(repeat_by, dim=0)
+
+    image = image.to(device=device, dtype=dtype)
+
+    if do_classifier_free_guidance and not guess_mode:
+        image = torch.cat([image] * 2)
+
+    return image
+
+class StableDiffusionLongPromptWeightingPipeline(StableDiffusionPipeline):
+    r"""
+    Pipeline for text-to-image generation using Stable Diffusion without tokens length limit, and support parsing
+    weighting in prompt.
+
+    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
+    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
+
+    Args:
+        vae ([`AutoencoderKL`]):
+            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
+        text_encoder ([`CLIPTextModel`]):
+            Frozen text-encoder. Stable Diffusion uses the text portion of
+            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
+            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
+        tokenizer (`CLIPTokenizer`):
+            Tokenizer of class
+            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
+        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
+        scheduler ([`SchedulerMixin`]):
+            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
+            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
+        safety_checker ([`StableDiffusionSafetyChecker`]):
+            Classification module that estimates whether generated images could be considered offensive or harmful.
+            Please, refer to the [model card](https://huggingface.co/CompVis/stable-diffusion-v1-4) for details.
+        feature_extractor ([`CLIPFeatureExtractor`]):
+            Model that extracts features from generated images to be used as inputs for the `safety_checker`.
+    """
+
+    # if version.parse(version.parse(diffusers.__version__).base_version) >= version.parse("0.9.0"):
+
+    def __init__(
+        self,
+        vae: AutoencoderKL,
+        text_encoder: CLIPTextModel,
+        tokenizer: CLIPTokenizer,
+        unet: UNet2DConditionModel,
+        scheduler: SchedulerMixin,
+        # clip_skip: int,
+        safety_checker: StableDiffusionSafetyChecker,
+        feature_extractor: CLIPFeatureExtractor,
+        requires_safety_checker: bool = True,
+        clip_skip: int = 1,
+    ):
+        super().__init__(
+            vae=vae,
+            text_encoder=text_encoder,
+            tokenizer=tokenizer,
+            unet=unet,
+            scheduler=scheduler,
+            safety_checker=safety_checker,
+            feature_extractor=feature_extractor,
+            requires_safety_checker=requires_safety_checker,
+        )
+        self.clip_skip = clip_skip
+        self.__init__additional__()
+
+    # else:
+    #     def __init__(
+    #         self,
+    #         vae: AutoencoderKL,
+    #         text_encoder: CLIPTextModel,
+    #         tokenizer: CLIPTokenizer,
+    #         unet: UNet2DConditionModel,
+    #         scheduler: SchedulerMixin,
+    #         safety_checker: StableDiffusionSafetyChecker,
+    #         feature_extractor: CLIPFeatureExtractor,
+    #     ):
+    #         super().__init__(
+    #             vae=vae,
+    #             text_encoder=text_encoder,
+    #             tokenizer=tokenizer,
+    #             unet=unet,
+    #             scheduler=scheduler,
+    #             safety_checker=safety_checker,
+    #             feature_extractor=feature_extractor,
+    #         )
+    #         self.__init__additional__()
+
+    def __init__additional__(self):
+        if not hasattr(self, "vae_scale_factor"):
+            setattr(self, "vae_scale_factor", 2 ** (len(self.vae.config.block_out_channels) - 1))
+
+    @property
+    def _execution_device(self):
+        r"""
+        Returns the device on which the pipeline's models will be executed. After calling
+        `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module
+        hooks.
+        """
+        if self.device != torch.device("meta") or not hasattr(self.unet, "_hf_hook"):
+            return self.device
+        for module in self.unet.modules():
+            if (
+                hasattr(module, "_hf_hook")
+                and hasattr(module._hf_hook, "execution_device")
+                and module._hf_hook.execution_device is not None
+            ):
+                return torch.device(module._hf_hook.execution_device)
+        return self.device
+
+    def _encode_prompt(
+        self,
+        prompt,
+        device,
+        num_images_per_prompt,
+        do_classifier_free_guidance,
+        negative_prompt,
+        max_embeddings_multiples,
+    ):
+        r"""
+        Encodes the prompt into text encoder hidden states.
+
+        Args:
+            prompt (`str` or `list(int)`):
+                prompt to be encoded
+            device: (`torch.device`):
+                torch device
+            num_images_per_prompt (`int`):
+                number of images that should be generated per prompt
+            do_classifier_free_guidance (`bool`):
+                whether to use classifier free guidance or not
+            negative_prompt (`str` or `List[str]`):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+        """
+        batch_size = len(prompt) if isinstance(prompt, list) else 1
+
+        if negative_prompt is None:
+            negative_prompt = [""] * batch_size
+        elif isinstance(negative_prompt, str):
+            negative_prompt = [negative_prompt] * batch_size
+        if batch_size != len(negative_prompt):
+            raise ValueError(
+                f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
+                f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
+                " the batch size of `prompt`."
+            )
+
+        text_embeddings, uncond_embeddings = get_weighted_text_embeddings(
+            pipe=self,
+            prompt=prompt,
+            uncond_prompt=negative_prompt if do_classifier_free_guidance else None,
+            max_embeddings_multiples=max_embeddings_multiples,
+            clip_skip=self.clip_skip,
+        )
+        bs_embed, seq_len, _ = text_embeddings.shape
+        text_embeddings = text_embeddings.repeat(1, num_images_per_prompt, 1)
+        text_embeddings = text_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)
+
+        if do_classifier_free_guidance:
+            bs_embed, seq_len, _ = uncond_embeddings.shape
+            uncond_embeddings = uncond_embeddings.repeat(1, num_images_per_prompt, 1)
+            uncond_embeddings = uncond_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)
+            text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
+
+        return text_embeddings
+
+    def check_inputs(self, prompt, height, width, strength, callback_steps):
+        if not isinstance(prompt, str) and not isinstance(prompt, list):
+            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
+
+        if strength < 0 or strength > 1:
+            raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
+
+        if height % 8 != 0 or width % 8 != 0:
+            print(height, width)
+            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
+
+        if (callback_steps is None) or (
+            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
+        ):
+            raise ValueError(
+                f"`callback_steps` has to be a positive integer but is {callback_steps} of type" f" {type(callback_steps)}."
+            )
+
+    def get_timesteps(self, num_inference_steps, strength, device, is_text2img):
+        if is_text2img:
+            return self.scheduler.timesteps.to(device), num_inference_steps
+        else:
+            # get the original timestep using init_timestep
+            offset = self.scheduler.config.get("steps_offset", 0)
+            init_timestep = int(num_inference_steps * strength) + offset
+            init_timestep = min(init_timestep, num_inference_steps)
+
+            t_start = max(num_inference_steps - init_timestep + offset, 0)
+            timesteps = self.scheduler.timesteps[t_start:].to(device)
+            return timesteps, num_inference_steps - t_start
+
+    def run_safety_checker(self, image, device, dtype):
+        if self.safety_checker is not None:
+            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)
+            image, has_nsfw_concept = self.safety_checker(images=image, clip_input=safety_checker_input.pixel_values.to(dtype))
+        else:
+            has_nsfw_concept = None
+        return image, has_nsfw_concept
+
+    def decode_latents(self, latents):
+        latents = 1 / 0.18215 * latents
+        image = self.vae.decode(latents).sample
+        image = (image / 2 + 0.5).clamp(0, 1)
+        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
+        image = image.cpu().permute(0, 2, 3, 1).float().numpy()
+        return image
+
+    def prepare_extra_step_kwargs(self, generator, eta):
+        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
+        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
+        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
+        # and should be between [0, 1]
+
+        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
+        extra_step_kwargs = {}
+        if accepts_eta:
+            extra_step_kwargs["eta"] = eta
+
+        # check if the scheduler accepts generator
+        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
+        if accepts_generator:
+            extra_step_kwargs["generator"] = generator
+        return extra_step_kwargs
+
+    def prepare_latents(self, image, timestep, batch_size, height, width, dtype, device, generator, latents=None):
+        if image is None:
+            shape = (
+                batch_size,
+                self.unet.in_channels,
+                height // self.vae_scale_factor,
+                width // self.vae_scale_factor,
+            )
+
+            if latents is None:
+                if device.type == "mps":
+                    # randn does not work reproducibly on mps
+                    latents = torch.randn(shape, generator=generator, device="cpu", dtype=dtype).to(device)
+                else:
+                    latents = torch.randn(shape, generator=generator, device=device, dtype=dtype)
+            else:
+                if latents.shape != shape:
+                    raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}")
+                latents = latents.to(device)
+
+            # scale the initial noise by the standard deviation required by the scheduler
+            latents = latents * self.scheduler.init_noise_sigma
+            return latents, None, None
+        else:
+            init_latent_dist = self.vae.encode(image).latent_dist
+            init_latents = init_latent_dist.sample(generator=generator)
+            init_latents = 0.18215 * init_latents
+            init_latents = torch.cat([init_latents] * batch_size, dim=0)
+            init_latents_orig = init_latents
+            shape = init_latents.shape
+
+            # add noise to latents using the timesteps
+            if device.type == "mps":
+                noise = torch.randn(shape, generator=generator, device="cpu", dtype=dtype).to(device)
+            else:
+                noise = torch.randn(shape, generator=generator, device=device, dtype=dtype)
+            latents = self.scheduler.add_noise(init_latents, noise, timestep)
+            return latents, init_latents_orig, noise
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        image: Union[torch.FloatTensor, PIL.Image.Image] = None,
+        mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None,
+        height: int = 512,
+        width: int = 512,
+        num_inference_steps: int = 50,
+        guidance_scale: float = 7.5,
+        strength: float = 0.8,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[torch.Generator] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        controlnet=None,
+        controlnet_image=None,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: int = 1,
+    ):
+        r"""
+        Function invoked when calling the pipeline for generation.
+
+        Args:
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process.
+            mask_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be
+                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a
+                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should
+                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.
+            height (`int`, *optional*, defaults to 512):
+                The height in pixels of the generated image.
+            width (`int`, *optional*, defaults to 512):
+                The width in pixels of the generated image.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.
+                `image` will be used as a starting point, adding more noise to it the larger the `strength`. The
+                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added
+                noise will be maximum and the denoising process will run for the full number of iterations specified in
+                `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            controlnet (`diffusers.ControlNetModel`, *optional*):
+                A controlnet model to be used for the inference. If not provided, controlnet will be disabled.
+            controlnet_image (`torch.FloatTensor` or `PIL.Image.Image`, *optional*):
+                `Image`, or tensor representing an image batch, to be used as the starting point for the controlnet
+                inference.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            is_cancelled_callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. If the function returns
+                `True`, the inference will be cancelled.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+
+        Returns:
+            `None` if cancelled by `is_cancelled_callback`,
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        if controlnet is not None and controlnet_image is None:
+            raise ValueError("controlnet_image must be provided if controlnet is not None.")
+
+        # 0. Default height and width to unet
+        height = height or self.unet.config.sample_size * self.vae_scale_factor
+        width = width or self.unet.config.sample_size * self.vae_scale_factor
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(prompt, height, width, strength, callback_steps)
+
+        # 2. Define call parameters
+        batch_size = 1 if isinstance(prompt, str) else len(prompt)
+        device = self._execution_device
+        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
+        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
+        # corresponds to doing no classifier free guidance.
+        do_classifier_free_guidance = guidance_scale > 1.0
+
+        # 3. Encode input prompt
+        text_embeddings = self._encode_prompt(
+            prompt,
+            device,
+            num_images_per_prompt,
+            do_classifier_free_guidance,
+            negative_prompt,
+            max_embeddings_multiples,
+        )
+        dtype = text_embeddings.dtype
+
+        # 4. Preprocess image and mask
+        if isinstance(image, PIL.Image.Image):
+            image = preprocess_image(image)
+        if image is not None:
+            image = image.to(device=self.device, dtype=dtype)
+        if isinstance(mask_image, PIL.Image.Image):
+            mask_image = preprocess_mask(mask_image, self.vae_scale_factor)
+        if mask_image is not None:
+            mask = mask_image.to(device=self.device, dtype=dtype)
+            mask = torch.cat([mask] * batch_size * num_images_per_prompt)
+        else:
+            mask = None
+
+        if controlnet_image is not None:
+            controlnet_image = prepare_controlnet_image(controlnet_image, width, height, batch_size, 1, self.device, controlnet.dtype, do_classifier_free_guidance, False)
+
+
+        # 5. set timesteps
+        self.scheduler.set_timesteps(num_inference_steps, device=device)
+        timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device, image is None)
+        latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
+
+        # 6. Prepare latent variables
+        latents, init_latents_orig, noise = self.prepare_latents(
+            image,
+            latent_timestep,
+            batch_size * num_images_per_prompt,
+            height,
+            width,
+            dtype,
+            device,
+            generator,
+            latents,
+        )
+
+        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+        # 8. Denoising loop
+        for i, t in enumerate(self.progress_bar(timesteps)):
+            # expand the latents if we are doing classifier free guidance
+            latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
+            latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+            unet_additional_args = {}
+            if controlnet is not None:
+                down_block_res_samples, mid_block_res_sample = controlnet(
+                    latent_model_input,
+                    t,
+                    encoder_hidden_states=text_embeddings,
+                    controlnet_cond=controlnet_image,
+                    conditioning_scale=1.0,
+                    guess_mode=False,
+                    return_dict=False,
+                )
+                unet_additional_args['down_block_additional_residuals'] = down_block_res_samples
+                unet_additional_args['mid_block_additional_residual'] = mid_block_res_sample
+
+            # predict the noise residual
+            noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings, **unet_additional_args).sample
+
+            # perform guidance
+            if do_classifier_free_guidance:
+                noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+            # compute the previous noisy sample x_t -> x_t-1
+            latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
+
+            if mask is not None:
+                # masking
+                init_latents_proper = self.scheduler.add_noise(init_latents_orig, noise, torch.tensor([t]))
+                latents = (init_latents_proper * mask) + (latents * (1 - mask))
+
+            # call the callback, if provided
+            if i % callback_steps == 0:
+                if callback is not None:
+                    callback(i, t, latents)
+                if is_cancelled_callback is not None and is_cancelled_callback():
+                    return None
+
+        # 9. Post-processing
+        image = self.decode_latents(latents)
+
+        # 10. Run safety checker
+        image, has_nsfw_concept = self.run_safety_checker(image, device, text_embeddings.dtype)
+
+        # 11. Convert to PIL
+        if output_type == "pil":
+            image = self.numpy_to_pil(image)
+
+        if not return_dict:
+            return image, has_nsfw_concept
+
+        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)
+
+    def text2img(
+        self,
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        height: int = 512,
+        width: int = 512,
+        num_inference_steps: int = 50,
+        guidance_scale: float = 7.5,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[torch.Generator] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: int = 1,
+    ):
+        r"""
+        Function for text-to-image generation.
+        Args:
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            height (`int`, *optional*, defaults to 512):
+                The height in pixels of the generated image.
+            width (`int`, *optional*, defaults to 512):
+                The width in pixels of the generated image.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            is_cancelled_callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. If the function returns
+                `True`, the inference will be cancelled.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        return self.__call__(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            height=height,
+            width=width,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            num_images_per_prompt=num_images_per_prompt,
+            eta=eta,
+            generator=generator,
+            latents=latents,
+            max_embeddings_multiples=max_embeddings_multiples,
+            output_type=output_type,
+            return_dict=return_dict,
+            callback=callback,
+            is_cancelled_callback=is_cancelled_callback,
+            callback_steps=callback_steps,
+        )
+
+    def img2img(
+        self,
+        image: Union[torch.FloatTensor, PIL.Image.Image],
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        strength: float = 0.8,
+        num_inference_steps: Optional[int] = 50,
+        guidance_scale: Optional[float] = 7.5,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: Optional[float] = 0.0,
+        generator: Optional[torch.Generator] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: int = 1,
+    ):
+        r"""
+        Function for image-to-image generation.
+        Args:
+            image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process.
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.
+                `image` will be used as a starting point, adding more noise to it the larger the `strength`. The
+                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added
+                noise will be maximum and the denoising process will run for the full number of iterations specified in
+                `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference. This parameter will be modulated by `strength`.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            is_cancelled_callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. If the function returns
+                `True`, the inference will be cancelled.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        return self.__call__(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image=image,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            strength=strength,
+            num_images_per_prompt=num_images_per_prompt,
+            eta=eta,
+            generator=generator,
+            max_embeddings_multiples=max_embeddings_multiples,
+            output_type=output_type,
+            return_dict=return_dict,
+            callback=callback,
+            is_cancelled_callback=is_cancelled_callback,
+            callback_steps=callback_steps,
+        )
+
+    def inpaint(
+        self,
+        image: Union[torch.FloatTensor, PIL.Image.Image],
+        mask_image: Union[torch.FloatTensor, PIL.Image.Image],
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        strength: float = 0.8,
+        num_inference_steps: Optional[int] = 50,
+        guidance_scale: Optional[float] = 7.5,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: Optional[float] = 0.0,
+        generator: Optional[torch.Generator] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: int = 1,
+    ):
+        r"""
+        Function for inpaint.
+        Args:
+            image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process. This is the image whose masked region will be inpainted.
+            mask_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be
+                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a
+                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should
+                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to inpaint the masked area. Must be between 0 and 1. When `strength`
+                is 1, the denoising process will be run on the masked area for the full number of iterations specified
+                in `num_inference_steps`. `image` will be used as a reference for the masked area, adding more
+                noise to that region the larger the `strength`. If `strength` is 0, no inpainting will occur.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The reference number of denoising steps. More denoising steps usually lead to a higher quality image at
+                the expense of slower inference. This parameter will be modulated by `strength`, as explained above.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            is_cancelled_callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. If the function returns
+                `True`, the inference will be cancelled.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        return self.__call__(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image=image,
+            mask_image=mask_image,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            strength=strength,
+            num_images_per_prompt=num_images_per_prompt,
+            eta=eta,
+            generator=generator,
+            max_embeddings_multiples=max_embeddings_multiples,
+            output_type=output_type,
+            return_dict=return_dict,
+            callback=callback,
+            is_cancelled_callback=is_cancelled_callback,
+            callback_steps=callback_steps,
+        )
diff --git a/library/merge_lora_gui.py b/library/merge_lora_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e6cf705c594e16f1ce712fd0f3eca2fc96e6b75
--- /dev/null
+++ b/library/merge_lora_gui.py
@@ -0,0 +1,291 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import (
+    get_saveasfilename_path,
+    get_any_file_path,
+    get_file_path,
+)
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def check_model(model):
+    if not model:
+        return True
+    if not os.path.isfile(model):
+        msgbox(f'The provided {model} is not a file')
+        return False
+    return True
+
+
+def verify_conditions(sd_model, lora_models):
+    lora_models_count = sum(1 for model in lora_models if model)
+    if sd_model and lora_models_count >= 1:
+        return True
+    elif not sd_model and lora_models_count >= 2:
+        return True
+    return False
+
+
+def merge_lora(
+    sd_model,
+    lora_a_model,
+    lora_b_model,
+    lora_c_model,
+    lora_d_model,
+    ratio_a,
+    ratio_b,
+    ratio_c,
+    ratio_d,
+    save_to,
+    precision,
+    save_precision,
+):
+    log.info('Merge model...')
+    models = [sd_model, lora_a_model, lora_b_model, lora_c_model, lora_d_model]
+    lora_models = models[1:]
+    ratios = [ratio_a, ratio_b, ratio_c, ratio_d]
+
+    if not verify_conditions(sd_model, lora_models):
+        log.info(
+            'Warning: Either provide at least one LoRa model along with the sd_model or at least two LoRa models if no sd_model is provided.'
+        )
+        return
+
+    for model in models:
+        if not check_model(model):
+            return
+
+    run_cmd = f'{PYTHON} "{os.path.join("networks","merge_lora.py")}"'
+    if sd_model:
+        run_cmd += f' --sd_model "{sd_model}"'
+    run_cmd += f' --save_precision {save_precision}'
+    run_cmd += f' --precision {precision}'
+    run_cmd += f' --save_to "{save_to}"'
+
+    # Create a space-separated string of non-empty models (from the second element onwards), enclosed in double quotes
+    models_cmd = ' '.join([f'"{model}"' for model in lora_models if model])
+
+    # Create a space-separated string of non-zero ratios corresponding to non-empty LoRa models
+    valid_ratios = [ratios[i] for i, model in enumerate(lora_models) if model]
+    ratios_cmd = ' '.join([str(ratio) for ratio in valid_ratios])
+
+    if models_cmd:
+        run_cmd += f' --models {models_cmd}'
+        run_cmd += f' --ratios {ratios_cmd}'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+    log.info('Done merging...')
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_merge_lora_tab(headless=False):
+    with gr.Tab('Merge LoRA'):
+        gr.Markdown(
+            'This utility can merge up to 4 LoRA together or alternatively merge up to 4 LoRA into a SD checkpoint.'
+        )
+
+        lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False)
+        lora_ext_name = gr.Textbox(value='LoRA model types', visible=False)
+        ckpt_ext = gr.Textbox(value='*.safetensors *.ckpt', visible=False)
+        ckpt_ext_name = gr.Textbox(value='SD model types', visible=False)
+
+        with gr.Row():
+            sd_model = gr.Textbox(
+                label='SD Model',
+                placeholder='(Optional) Stable Diffusion model',
+                interactive=True,
+                info='Provide a SD file path IF you want to merge it with LoRA files',
+            )
+            sd_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            sd_model_file.click(
+                get_file_path,
+                inputs=[sd_model, ckpt_ext, ckpt_ext_name],
+                outputs=sd_model,
+                show_progress=False,
+            )
+
+        with gr.Row():
+            lora_a_model = gr.Textbox(
+                label='LoRA model "A"',
+                placeholder='Path to the LoRA A model',
+                interactive=True,
+            )
+            button_lora_a_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_a_model_file.click(
+                get_file_path,
+                inputs=[lora_a_model, lora_ext, lora_ext_name],
+                outputs=lora_a_model,
+                show_progress=False,
+            )
+
+            lora_b_model = gr.Textbox(
+                label='LoRA model "B"',
+                placeholder='Path to the LoRA B model',
+                interactive=True,
+            )
+            button_lora_b_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_b_model_file.click(
+                get_file_path,
+                inputs=[lora_b_model, lora_ext, lora_ext_name],
+                outputs=lora_b_model,
+                show_progress=False,
+            )
+
+        with gr.Row():
+            ratio_a = gr.Slider(
+                label='Model A merge ratio (eg: 0.5 mean 50%)',
+                minimum=0,
+                maximum=1,
+                step=0.01,
+                value=0.0,
+                interactive=True,
+            )
+
+            ratio_b = gr.Slider(
+                label='Model B merge ratio (eg: 0.5 mean 50%)',
+                minimum=0,
+                maximum=1,
+                step=0.01,
+                value=0.0,
+                interactive=True,
+            )
+
+        with gr.Row():
+            lora_c_model = gr.Textbox(
+                label='LoRA model "C"',
+                placeholder='Path to the LoRA C model',
+                interactive=True,
+            )
+            button_lora_c_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_c_model_file.click(
+                get_file_path,
+                inputs=[lora_c_model, lora_ext, lora_ext_name],
+                outputs=lora_c_model,
+                show_progress=False,
+            )
+
+            lora_d_model = gr.Textbox(
+                label='LoRA model "D"',
+                placeholder='Path to the LoRA D model',
+                interactive=True,
+            )
+            button_lora_d_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_d_model_file.click(
+                get_file_path,
+                inputs=[lora_d_model, lora_ext, lora_ext_name],
+                outputs=lora_d_model,
+                show_progress=False,
+            )
+
+        with gr.Row():
+            ratio_c = gr.Slider(
+                label='Model C merge ratio (eg: 0.5 mean 50%)',
+                minimum=0,
+                maximum=1,
+                step=0.01,
+                value=0.0,
+                interactive=True,
+            )
+
+            ratio_d = gr.Slider(
+                label='Model D merge ratio (eg: 0.5 mean 50%)',
+                minimum=0,
+                maximum=1,
+                step=0.01,
+                value=0.0,
+                interactive=True,
+            )
+
+        with gr.Row():
+            save_to = gr.Textbox(
+                label='Save to',
+                placeholder='path for the file to save...',
+                interactive=True,
+            )
+            button_save_to = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_save_to.click(
+                get_saveasfilename_path,
+                inputs=[save_to, lora_ext, lora_ext_name],
+                outputs=save_to,
+                show_progress=False,
+            )
+            precision = gr.Dropdown(
+                label='Merge precision',
+                choices=['fp16', 'bf16', 'float'],
+                value='float',
+                interactive=True,
+            )
+            save_precision = gr.Dropdown(
+                label='Save precision',
+                choices=['fp16', 'bf16', 'float'],
+                value='fp16',
+                interactive=True,
+            )
+
+        merge_button = gr.Button('Merge model')
+
+        merge_button.click(
+            merge_lora,
+            inputs=[
+                sd_model,
+                lora_a_model,
+                lora_b_model,
+                lora_c_model,
+                lora_d_model,
+                ratio_a,
+                ratio_b,
+                ratio_c,
+                ratio_d,
+                save_to,
+                precision,
+                save_precision,
+            ],
+            show_progress=False,
+        )
diff --git a/library/merge_lycoris_gui.py b/library/merge_lycoris_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..7d56f1e07f92f5fa076dd288f0dbd49c9772a79d
--- /dev/null
+++ b/library/merge_lycoris_gui.py
@@ -0,0 +1,174 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import (
+    get_saveasfilename_path,
+    get_file_path,
+)
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def merge_lycoris(
+    base_model,
+    lycoris_model,
+    weight,
+    output_name,
+    dtype,
+    device,
+    is_v2,
+):
+    log.info('Merge model...')
+
+    run_cmd = f'{PYTHON} "{os.path.join("tools","merge_lycoris.py")}"'
+    run_cmd += f' "{base_model}"'
+    run_cmd += f' "{lycoris_model}"'
+    run_cmd += f' "{output_name}"'
+    run_cmd += f' --weight {weight}'
+    run_cmd += f' --device {device}'
+    run_cmd += f' --dtype {dtype}'
+    if is_v2:
+        run_cmd += f' --is_v2'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+    log.info('Done merging...')
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_merge_lycoris_tab(headless=False):
+    with gr.Tab('Merge LyCORIS'):
+        gr.Markdown(
+            'This utility can merge a LyCORIS model into a SD checkpoint.'
+        )
+
+        lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False)
+        lora_ext_name = gr.Textbox(value='LoRA model types', visible=False)
+        ckpt_ext = gr.Textbox(value='*.safetensors *.ckpt', visible=False)
+        ckpt_ext_name = gr.Textbox(value='SD model types', visible=False)
+
+        with gr.Row():
+            base_model = gr.Textbox(
+                label='SD Model',
+                placeholder='(Optional) Stable Diffusion base model',
+                interactive=True,
+                info='Provide a SD file path that you want to merge with the LyCORIS file',
+            )
+            base_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            base_model_file.click(
+                get_file_path,
+                inputs=[base_model, ckpt_ext, ckpt_ext_name],
+                outputs=base_model,
+                show_progress=False,
+            )
+
+        with gr.Row():
+            lycoris_model = gr.Textbox(
+                label='LyCORIS model',
+                placeholder='Path to the LyCORIS model',
+                interactive=True,
+            )
+            button_lycoris_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lycoris_model_file.click(
+                get_file_path,
+                inputs=[lycoris_model, lora_ext, lora_ext_name],
+                outputs=lycoris_model,
+                show_progress=False,
+            )
+
+        with gr.Row():
+            weight = gr.Slider(
+                label='Model A merge ratio (eg: 0.5 mean 50%)',
+                minimum=0,
+                maximum=1,
+                step=0.01,
+                value=1.0,
+                interactive=True,
+            )
+
+        with gr.Row():
+            output_name = gr.Textbox(
+                label='Save to',
+                placeholder='path for the checkpoint file to save...',
+                interactive=True,
+            )
+            button_output_name = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_output_name.click(
+                get_saveasfilename_path,
+                inputs=[output_name, lora_ext, lora_ext_name],
+                outputs=output_name,
+                show_progress=False,
+            )
+            dtype = gr.Dropdown(
+                label='Save dtype',
+                choices=[
+                    'float',
+                    'float16',
+                    'float32',
+                    'float64',
+                    'bfloat',
+                    'bfloat16',
+                ],
+                value='float16',
+                interactive=True,
+            )
+
+            device = gr.Dropdown(
+                label='Device',
+                choices=[
+                    'cpu',
+                    #  'cuda',
+                ],
+                value='cpu',
+                interactive=True,
+            )
+
+            is_v2 = gr.Checkbox(label='is v2', value=False, interactive=True)
+
+        merge_button = gr.Button('Merge model')
+
+        merge_button.click(
+            merge_lycoris,
+            inputs=[
+                base_model,
+                lycoris_model,
+                weight,
+                output_name,
+                dtype,
+                device,
+                is_v2,
+            ],
+            show_progress=False,
+        )
diff --git a/library/model_util.py b/library/model_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..938b76929d7d6c763988f49f7ce84ae2dc70f6cc
--- /dev/null
+++ b/library/model_util.py
@@ -0,0 +1,1333 @@
+# v1: split from train_db_fixed.py.
+# v2: support safetensors
+
+import math
+import os
+import torch
+import diffusers
+from transformers import CLIPTextModel, CLIPTokenizer, CLIPTextConfig, logging
+from diffusers import AutoencoderKL, DDIMScheduler, StableDiffusionPipeline  # , UNet2DConditionModel
+from safetensors.torch import load_file, save_file
+from library.original_unet import UNet2DConditionModel
+
+# DiffUsers版StableDiffusionのモデルパラメータ
+NUM_TRAIN_TIMESTEPS = 1000
+BETA_START = 0.00085
+BETA_END = 0.0120
+
+UNET_PARAMS_MODEL_CHANNELS = 320
+UNET_PARAMS_CHANNEL_MULT = [1, 2, 4, 4]
+UNET_PARAMS_ATTENTION_RESOLUTIONS = [4, 2, 1]
+UNET_PARAMS_IMAGE_SIZE = 64  # fixed from old invalid value `32`
+UNET_PARAMS_IN_CHANNELS = 4
+UNET_PARAMS_OUT_CHANNELS = 4
+UNET_PARAMS_NUM_RES_BLOCKS = 2
+UNET_PARAMS_CONTEXT_DIM = 768
+UNET_PARAMS_NUM_HEADS = 8
+# UNET_PARAMS_USE_LINEAR_PROJECTION = False
+
+VAE_PARAMS_Z_CHANNELS = 4
+VAE_PARAMS_RESOLUTION = 256
+VAE_PARAMS_IN_CHANNELS = 3
+VAE_PARAMS_OUT_CH = 3
+VAE_PARAMS_CH = 128
+VAE_PARAMS_CH_MULT = [1, 2, 4, 4]
+VAE_PARAMS_NUM_RES_BLOCKS = 2
+
+# V2
+V2_UNET_PARAMS_ATTENTION_HEAD_DIM = [5, 10, 20, 20]
+V2_UNET_PARAMS_CONTEXT_DIM = 1024
+# V2_UNET_PARAMS_USE_LINEAR_PROJECTION = True
+
+# Diffusersの設定を読み込むための参照モデル
+DIFFUSERS_REF_MODEL_ID_V1 = "runwayml/stable-diffusion-v1-5"
+DIFFUSERS_REF_MODEL_ID_V2 = "stabilityai/stable-diffusion-2-1"
+
+
+# region StableDiffusion->Diffusersの変換コード
+# convert_original_stable_diffusion_to_diffusers をコピーして修正している（ASL 2.0）
+
+
+def shave_segments(path, n_shave_prefix_segments=1):
+    """
+    Removes segments. Positive values shave the first segments, negative shave the last segments.
+    """
+    if n_shave_prefix_segments >= 0:
+        return ".".join(path.split(".")[n_shave_prefix_segments:])
+    else:
+        return ".".join(path.split(".")[:n_shave_prefix_segments])
+
+
+def renew_resnet_paths(old_list, n_shave_prefix_segments=0):
+    """
+    Updates paths inside resnets to the new naming scheme (local renaming)
+    """
+    mapping = []
+    for old_item in old_list:
+        new_item = old_item.replace("in_layers.0", "norm1")
+        new_item = new_item.replace("in_layers.2", "conv1")
+
+        new_item = new_item.replace("out_layers.0", "norm2")
+        new_item = new_item.replace("out_layers.3", "conv2")
+
+        new_item = new_item.replace("emb_layers.1", "time_emb_proj")
+        new_item = new_item.replace("skip_connection", "conv_shortcut")
+
+        new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)
+
+        mapping.append({"old": old_item, "new": new_item})
+
+    return mapping
+
+
+def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0):
+    """
+    Updates paths inside resnets to the new naming scheme (local renaming)
+    """
+    mapping = []
+    for old_item in old_list:
+        new_item = old_item
+
+        new_item = new_item.replace("nin_shortcut", "conv_shortcut")
+        new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)
+
+        mapping.append({"old": old_item, "new": new_item})
+
+    return mapping
+
+
+def renew_attention_paths(old_list, n_shave_prefix_segments=0):
+    """
+    Updates paths inside attentions to the new naming scheme (local renaming)
+    """
+    mapping = []
+    for old_item in old_list:
+        new_item = old_item
+
+        #         new_item = new_item.replace('norm.weight', 'group_norm.weight')
+        #         new_item = new_item.replace('norm.bias', 'group_norm.bias')
+
+        #         new_item = new_item.replace('proj_out.weight', 'proj_attn.weight')
+        #         new_item = new_item.replace('proj_out.bias', 'proj_attn.bias')
+
+        #         new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)
+
+        mapping.append({"old": old_item, "new": new_item})
+
+    return mapping
+
+
+def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0):
+    """
+    Updates paths inside attentions to the new naming scheme (local renaming)
+    """
+    mapping = []
+    for old_item in old_list:
+        new_item = old_item
+
+        new_item = new_item.replace("norm.weight", "group_norm.weight")
+        new_item = new_item.replace("norm.bias", "group_norm.bias")
+
+        if diffusers.__version__ < "0.17.0":
+            new_item = new_item.replace("q.weight", "query.weight")
+            new_item = new_item.replace("q.bias", "query.bias")
+
+            new_item = new_item.replace("k.weight", "key.weight")
+            new_item = new_item.replace("k.bias", "key.bias")
+
+            new_item = new_item.replace("v.weight", "value.weight")
+            new_item = new_item.replace("v.bias", "value.bias")
+
+            new_item = new_item.replace("proj_out.weight", "proj_attn.weight")
+            new_item = new_item.replace("proj_out.bias", "proj_attn.bias")
+        else:
+            new_item = new_item.replace("q.weight", "to_q.weight")
+            new_item = new_item.replace("q.bias", "to_q.bias")
+
+            new_item = new_item.replace("k.weight", "to_k.weight")
+            new_item = new_item.replace("k.bias", "to_k.bias")
+
+            new_item = new_item.replace("v.weight", "to_v.weight")
+            new_item = new_item.replace("v.bias", "to_v.bias")
+
+            new_item = new_item.replace("proj_out.weight", "to_out.0.weight")
+            new_item = new_item.replace("proj_out.bias", "to_out.0.bias")
+
+        new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments)
+
+        mapping.append({"old": old_item, "new": new_item})
+
+    return mapping
+
+
+def assign_to_checkpoint(
+    paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None
+):
+    """
+    This does the final conversion step: take locally converted weights and apply a global renaming
+    to them. It splits attention layers, and takes into account additional replacements
+    that may arise.
+
+    Assigns the weights to the new checkpoint.
+    """
+    assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys."
+
+    # Splits the attention layers into three variables.
+    if attention_paths_to_split is not None:
+        for path, path_map in attention_paths_to_split.items():
+            old_tensor = old_checkpoint[path]
+            channels = old_tensor.shape[0] // 3
+
+            target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1)
+
+            num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3
+
+            old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:])
+            query, key, value = old_tensor.split(channels // num_heads, dim=1)
+
+            checkpoint[path_map["query"]] = query.reshape(target_shape)
+            checkpoint[path_map["key"]] = key.reshape(target_shape)
+            checkpoint[path_map["value"]] = value.reshape(target_shape)
+
+    for path in paths:
+        new_path = path["new"]
+
+        # These have already been assigned
+        if attention_paths_to_split is not None and new_path in attention_paths_to_split:
+            continue
+
+        # Global renaming happens here
+        new_path = new_path.replace("middle_block.0", "mid_block.resnets.0")
+        new_path = new_path.replace("middle_block.1", "mid_block.attentions.0")
+        new_path = new_path.replace("middle_block.2", "mid_block.resnets.1")
+
+        if additional_replacements is not None:
+            for replacement in additional_replacements:
+                new_path = new_path.replace(replacement["old"], replacement["new"])
+
+        # proj_attn.weight has to be converted from conv 1D to linear
+        reshaping = False
+        if diffusers.__version__ < "0.17.0":
+            if "proj_attn.weight" in new_path:
+                reshaping = True
+        else:
+            if ".attentions." in new_path and ".0.to_" in new_path and old_checkpoint[path["old"]].ndim > 2:
+                reshaping = True
+
+        if reshaping:
+            checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0, 0]
+        else:
+            checkpoint[new_path] = old_checkpoint[path["old"]]
+
+
+def conv_attn_to_linear(checkpoint):
+    keys = list(checkpoint.keys())
+    attn_keys = ["query.weight", "key.weight", "value.weight"]
+    for key in keys:
+        if ".".join(key.split(".")[-2:]) in attn_keys:
+            if checkpoint[key].ndim > 2:
+                checkpoint[key] = checkpoint[key][:, :, 0, 0]
+        elif "proj_attn.weight" in key:
+            if checkpoint[key].ndim > 2:
+                checkpoint[key] = checkpoint[key][:, :, 0]
+
+
+def linear_transformer_to_conv(checkpoint):
+    keys = list(checkpoint.keys())
+    tf_keys = ["proj_in.weight", "proj_out.weight"]
+    for key in keys:
+        if ".".join(key.split(".")[-2:]) in tf_keys:
+            if checkpoint[key].ndim == 2:
+                checkpoint[key] = checkpoint[key].unsqueeze(2).unsqueeze(2)
+
+
+def convert_ldm_unet_checkpoint(v2, checkpoint, config):
+    """
+    Takes a state dict and a config, and returns a converted checkpoint.
+    """
+
+    # extract state_dict for UNet
+    unet_state_dict = {}
+    unet_key = "model.diffusion_model."
+    keys = list(checkpoint.keys())
+    for key in keys:
+        if key.startswith(unet_key):
+            unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key)
+
+    new_checkpoint = {}
+
+    new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"]
+    new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"]
+    new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"]
+    new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"]
+
+    new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"]
+    new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"]
+
+    new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"]
+    new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"]
+    new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"]
+    new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"]
+
+    # Retrieves the keys for the input blocks only
+    num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer})
+    input_blocks = {
+        layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}." in key] for layer_id in range(num_input_blocks)
+    }
+
+    # Retrieves the keys for the middle blocks only
+    num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer})
+    middle_blocks = {
+        layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}." in key] for layer_id in range(num_middle_blocks)
+    }
+
+    # Retrieves the keys for the output blocks only
+    num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer})
+    output_blocks = {
+        layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}." in key] for layer_id in range(num_output_blocks)
+    }
+
+    for i in range(1, num_input_blocks):
+        block_id = (i - 1) // (config["layers_per_block"] + 1)
+        layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1)
+
+        resnets = [key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key]
+        attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key]
+
+        if f"input_blocks.{i}.0.op.weight" in unet_state_dict:
+            new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop(
+                f"input_blocks.{i}.0.op.weight"
+            )
+            new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop(f"input_blocks.{i}.0.op.bias")
+
+        paths = renew_resnet_paths(resnets)
+        meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"}
+        assign_to_checkpoint(paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config)
+
+        if len(attentions):
+            paths = renew_attention_paths(attentions)
+            meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"}
+            assign_to_checkpoint(paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config)
+
+    resnet_0 = middle_blocks[0]
+    attentions = middle_blocks[1]
+    resnet_1 = middle_blocks[2]
+
+    resnet_0_paths = renew_resnet_paths(resnet_0)
+    assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config)
+
+    resnet_1_paths = renew_resnet_paths(resnet_1)
+    assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config)
+
+    attentions_paths = renew_attention_paths(attentions)
+    meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"}
+    assign_to_checkpoint(attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config)
+
+    for i in range(num_output_blocks):
+        block_id = i // (config["layers_per_block"] + 1)
+        layer_in_block_id = i % (config["layers_per_block"] + 1)
+        output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]]
+        output_block_list = {}
+
+        for layer in output_block_layers:
+            layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1)
+            if layer_id in output_block_list:
+                output_block_list[layer_id].append(layer_name)
+            else:
+                output_block_list[layer_id] = [layer_name]
+
+        if len(output_block_list) > 1:
+            resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key]
+            attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key]
+
+            resnet_0_paths = renew_resnet_paths(resnets)
+            paths = renew_resnet_paths(resnets)
+
+            meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"}
+            assign_to_checkpoint(paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config)
+
+            # オリジナル：
+            # if ["conv.weight", "conv.bias"] in output_block_list.values():
+            #   index = list(output_block_list.values()).index(["conv.weight", "conv.bias"])
+
+            # biasとweightの順番に依存しないようにする：もっといいやり方がありそうだが
+            for l in output_block_list.values():
+                l.sort()
+
+            if ["conv.bias", "conv.weight"] in output_block_list.values():
+                index = list(output_block_list.values()).index(["conv.bias", "conv.weight"])
+                new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[
+                    f"output_blocks.{i}.{index}.conv.bias"
+                ]
+                new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[
+                    f"output_blocks.{i}.{index}.conv.weight"
+                ]
+
+                # Clear attentions as they have been attributed above.
+                if len(attentions) == 2:
+                    attentions = []
+
+            if len(attentions):
+                paths = renew_attention_paths(attentions)
+                meta_path = {
+                    "old": f"output_blocks.{i}.1",
+                    "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}",
+                }
+                assign_to_checkpoint(paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config)
+        else:
+            resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1)
+            for path in resnet_0_paths:
+                old_path = ".".join(["output_blocks", str(i), path["old"]])
+                new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]])
+
+                new_checkpoint[new_path] = unet_state_dict[old_path]
+
+    # SDのv2では1*1のconv2dがlinearに変わっている
+    # 誤って Diffusers 側を conv2d のままにしてしまったので、変換必要
+    if v2 and not config.get("use_linear_projection", False):
+        linear_transformer_to_conv(new_checkpoint)
+
+    return new_checkpoint
+
+
+def convert_ldm_vae_checkpoint(checkpoint, config):
+    # extract state dict for VAE
+    vae_state_dict = {}
+    vae_key = "first_stage_model."
+    keys = list(checkpoint.keys())
+    for key in keys:
+        if key.startswith(vae_key):
+            vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key)
+    # if len(vae_state_dict) == 0:
+    #   # 渡されたcheckpointは.ckptから読み込んだcheckpointではなくvaeのstate_dict
+    #   vae_state_dict = checkpoint
+
+    new_checkpoint = {}
+
+    new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"]
+    new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"]
+    new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"]
+    new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"]
+    new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"]
+    new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"]
+
+    new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"]
+    new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"]
+    new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"]
+    new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"]
+    new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"]
+    new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"]
+
+    new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"]
+    new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"]
+    new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"]
+    new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"]
+
+    # Retrieves the keys for the encoder down blocks only
+    num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer})
+    down_blocks = {layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks)}
+
+    # Retrieves the keys for the decoder up blocks only
+    num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer})
+    up_blocks = {layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks)}
+
+    for i in range(num_down_blocks):
+        resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key]
+
+        if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict:
+            new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop(
+                f"encoder.down.{i}.downsample.conv.weight"
+            )
+            new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop(
+                f"encoder.down.{i}.downsample.conv.bias"
+            )
+
+        paths = renew_vae_resnet_paths(resnets)
+        meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"}
+        assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
+
+    mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key]
+    num_mid_res_blocks = 2
+    for i in range(1, num_mid_res_blocks + 1):
+        resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key]
+
+        paths = renew_vae_resnet_paths(resnets)
+        meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"}
+        assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
+
+    mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key]
+    paths = renew_vae_attention_paths(mid_attentions)
+    meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"}
+    assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
+    conv_attn_to_linear(new_checkpoint)
+
+    for i in range(num_up_blocks):
+        block_id = num_up_blocks - 1 - i
+        resnets = [key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key]
+
+        if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict:
+            new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[
+                f"decoder.up.{block_id}.upsample.conv.weight"
+            ]
+            new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[
+                f"decoder.up.{block_id}.upsample.conv.bias"
+            ]
+
+        paths = renew_vae_resnet_paths(resnets)
+        meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"}
+        assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
+
+    mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key]
+    num_mid_res_blocks = 2
+    for i in range(1, num_mid_res_blocks + 1):
+        resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key]
+
+        paths = renew_vae_resnet_paths(resnets)
+        meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"}
+        assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
+
+    mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key]
+    paths = renew_vae_attention_paths(mid_attentions)
+    meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"}
+    assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config)
+    conv_attn_to_linear(new_checkpoint)
+    return new_checkpoint
+
+
+def create_unet_diffusers_config(v2, use_linear_projection_in_v2=False):
+    """
+    Creates a config for the diffusers based on the config of the LDM model.
+    """
+    # unet_params = original_config.model.params.unet_config.params
+
+    block_out_channels = [UNET_PARAMS_MODEL_CHANNELS * mult for mult in UNET_PARAMS_CHANNEL_MULT]
+
+    down_block_types = []
+    resolution = 1
+    for i in range(len(block_out_channels)):
+        block_type = "CrossAttnDownBlock2D" if resolution in UNET_PARAMS_ATTENTION_RESOLUTIONS else "DownBlock2D"
+        down_block_types.append(block_type)
+        if i != len(block_out_channels) - 1:
+            resolution *= 2
+
+    up_block_types = []
+    for i in range(len(block_out_channels)):
+        block_type = "CrossAttnUpBlock2D" if resolution in UNET_PARAMS_ATTENTION_RESOLUTIONS else "UpBlock2D"
+        up_block_types.append(block_type)
+        resolution //= 2
+
+    config = dict(
+        sample_size=UNET_PARAMS_IMAGE_SIZE,
+        in_channels=UNET_PARAMS_IN_CHANNELS,
+        out_channels=UNET_PARAMS_OUT_CHANNELS,
+        down_block_types=tuple(down_block_types),
+        up_block_types=tuple(up_block_types),
+        block_out_channels=tuple(block_out_channels),
+        layers_per_block=UNET_PARAMS_NUM_RES_BLOCKS,
+        cross_attention_dim=UNET_PARAMS_CONTEXT_DIM if not v2 else V2_UNET_PARAMS_CONTEXT_DIM,
+        attention_head_dim=UNET_PARAMS_NUM_HEADS if not v2 else V2_UNET_PARAMS_ATTENTION_HEAD_DIM,
+        # use_linear_projection=UNET_PARAMS_USE_LINEAR_PROJECTION if not v2 else V2_UNET_PARAMS_USE_LINEAR_PROJECTION,
+    )
+    if v2 and use_linear_projection_in_v2:
+        config["use_linear_projection"] = True
+
+    return config
+
+
+def create_vae_diffusers_config():
+    """
+    Creates a config for the diffusers based on the config of the LDM model.
+    """
+    # vae_params = original_config.model.params.first_stage_config.params.ddconfig
+    # _ = original_config.model.params.first_stage_config.params.embed_dim
+    block_out_channels = [VAE_PARAMS_CH * mult for mult in VAE_PARAMS_CH_MULT]
+    down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels)
+    up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels)
+
+    config = dict(
+        sample_size=VAE_PARAMS_RESOLUTION,
+        in_channels=VAE_PARAMS_IN_CHANNELS,
+        out_channels=VAE_PARAMS_OUT_CH,
+        down_block_types=tuple(down_block_types),
+        up_block_types=tuple(up_block_types),
+        block_out_channels=tuple(block_out_channels),
+        latent_channels=VAE_PARAMS_Z_CHANNELS,
+        layers_per_block=VAE_PARAMS_NUM_RES_BLOCKS,
+    )
+    return config
+
+
+def convert_ldm_clip_checkpoint_v1(checkpoint):
+    keys = list(checkpoint.keys())
+    text_model_dict = {}
+    for key in keys:
+        if key.startswith("cond_stage_model.transformer"):
+            text_model_dict[key[len("cond_stage_model.transformer.") :]] = checkpoint[key]
+    return text_model_dict
+
+
+def convert_ldm_clip_checkpoint_v2(checkpoint, max_length):
+    # 嫌になるくらい違うぞ！
+    def convert_key(key):
+        if not key.startswith("cond_stage_model"):
+            return None
+
+        # common conversion
+        key = key.replace("cond_stage_model.model.transformer.", "text_model.encoder.")
+        key = key.replace("cond_stage_model.model.", "text_model.")
+
+        if "resblocks" in key:
+            # resblocks conversion
+            key = key.replace(".resblocks.", ".layers.")
+            if ".ln_" in key:
+                key = key.replace(".ln_", ".layer_norm")
+            elif ".mlp." in key:
+                key = key.replace(".c_fc.", ".fc1.")
+                key = key.replace(".c_proj.", ".fc2.")
+            elif ".attn.out_proj" in key:
+                key = key.replace(".attn.out_proj.", ".self_attn.out_proj.")
+            elif ".attn.in_proj" in key:
+                key = None  # 特殊なので後で処理する
+            else:
+                raise ValueError(f"unexpected key in SD: {key}")
+        elif ".positional_embedding" in key:
+            key = key.replace(".positional_embedding", ".embeddings.position_embedding.weight")
+        elif ".text_projection" in key:
+            key = None  # 使われない???
+        elif ".logit_scale" in key:
+            key = None  # 使われない???
+        elif ".token_embedding" in key:
+            key = key.replace(".token_embedding.weight", ".embeddings.token_embedding.weight")
+        elif ".ln_final" in key:
+            key = key.replace(".ln_final", ".final_layer_norm")
+        return key
+
+    keys = list(checkpoint.keys())
+    new_sd = {}
+    for key in keys:
+        # remove resblocks 23
+        if ".resblocks.23." in key:
+            continue
+        new_key = convert_key(key)
+        if new_key is None:
+            continue
+        new_sd[new_key] = checkpoint[key]
+
+    # attnの変換
+    for key in keys:
+        if ".resblocks.23." in key:
+            continue
+        if ".resblocks" in key and ".attn.in_proj_" in key:
+            # 三つに分割
+            values = torch.chunk(checkpoint[key], 3)
+
+            key_suffix = ".weight" if "weight" in key else ".bias"
+            key_pfx = key.replace("cond_stage_model.model.transformer.resblocks.", "text_model.encoder.layers.")
+            key_pfx = key_pfx.replace("_weight", "")
+            key_pfx = key_pfx.replace("_bias", "")
+            key_pfx = key_pfx.replace(".attn.in_proj", ".self_attn.")
+            new_sd[key_pfx + "q_proj" + key_suffix] = values[0]
+            new_sd[key_pfx + "k_proj" + key_suffix] = values[1]
+            new_sd[key_pfx + "v_proj" + key_suffix] = values[2]
+
+    # rename or add position_ids
+    ANOTHER_POSITION_IDS_KEY = "text_model.encoder.text_model.embeddings.position_ids"
+    if ANOTHER_POSITION_IDS_KEY in new_sd:
+        # waifu diffusion v1.4
+        position_ids = new_sd[ANOTHER_POSITION_IDS_KEY]
+        del new_sd[ANOTHER_POSITION_IDS_KEY]
+    else:
+        position_ids = torch.Tensor([list(range(max_length))]).to(torch.int64)
+
+    new_sd["text_model.embeddings.position_ids"] = position_ids
+    return new_sd
+
+
+# endregion
+
+
+# region Diffusers->StableDiffusion の変換コード
+# convert_diffusers_to_original_stable_diffusion をコピーして修正している（ASL 2.0）
+
+
+def conv_transformer_to_linear(checkpoint):
+    keys = list(checkpoint.keys())
+    tf_keys = ["proj_in.weight", "proj_out.weight"]
+    for key in keys:
+        if ".".join(key.split(".")[-2:]) in tf_keys:
+            if checkpoint[key].ndim > 2:
+                checkpoint[key] = checkpoint[key][:, :, 0, 0]
+
+
+def convert_unet_state_dict_to_sd(v2, unet_state_dict):
+    unet_conversion_map = [
+        # (stable-diffusion, HF Diffusers)
+        ("time_embed.0.weight", "time_embedding.linear_1.weight"),
+        ("time_embed.0.bias", "time_embedding.linear_1.bias"),
+        ("time_embed.2.weight", "time_embedding.linear_2.weight"),
+        ("time_embed.2.bias", "time_embedding.linear_2.bias"),
+        ("input_blocks.0.0.weight", "conv_in.weight"),
+        ("input_blocks.0.0.bias", "conv_in.bias"),
+        ("out.0.weight", "conv_norm_out.weight"),
+        ("out.0.bias", "conv_norm_out.bias"),
+        ("out.2.weight", "conv_out.weight"),
+        ("out.2.bias", "conv_out.bias"),
+    ]
+
+    unet_conversion_map_resnet = [
+        # (stable-diffusion, HF Diffusers)
+        ("in_layers.0", "norm1"),
+        ("in_layers.2", "conv1"),
+        ("out_layers.0", "norm2"),
+        ("out_layers.3", "conv2"),
+        ("emb_layers.1", "time_emb_proj"),
+        ("skip_connection", "conv_shortcut"),
+    ]
+
+    unet_conversion_map_layer = []
+    for i in range(4):
+        # loop over downblocks/upblocks
+
+        for j in range(2):
+            # loop over resnets/attentions for downblocks
+            hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}."
+            sd_down_res_prefix = f"input_blocks.{3*i + j + 1}.0."
+            unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix))
+
+            if i < 3:
+                # no attention layers in down_blocks.3
+                hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}."
+                sd_down_atn_prefix = f"input_blocks.{3*i + j + 1}.1."
+                unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix))
+
+        for j in range(3):
+            # loop over resnets/attentions for upblocks
+            hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}."
+            sd_up_res_prefix = f"output_blocks.{3*i + j}.0."
+            unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix))
+
+            if i > 0:
+                # no attention layers in up_blocks.0
+                hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}."
+                sd_up_atn_prefix = f"output_blocks.{3*i + j}.1."
+                unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix))
+
+        if i < 3:
+            # no downsample in down_blocks.3
+            hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv."
+            sd_downsample_prefix = f"input_blocks.{3*(i+1)}.0.op."
+            unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix))
+
+            # no upsample in up_blocks.3
+            hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0."
+            sd_upsample_prefix = f"output_blocks.{3*i + 2}.{1 if i == 0 else 2}."
+            unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix))
+
+    hf_mid_atn_prefix = "mid_block.attentions.0."
+    sd_mid_atn_prefix = "middle_block.1."
+    unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix))
+
+    for j in range(2):
+        hf_mid_res_prefix = f"mid_block.resnets.{j}."
+        sd_mid_res_prefix = f"middle_block.{2*j}."
+        unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix))
+
+    # buyer beware: this is a *brittle* function,
+    # and correct output requires that all of these pieces interact in
+    # the exact order in which I have arranged them.
+    mapping = {k: k for k in unet_state_dict.keys()}
+    for sd_name, hf_name in unet_conversion_map:
+        mapping[hf_name] = sd_name
+    for k, v in mapping.items():
+        if "resnets" in k:
+            for sd_part, hf_part in unet_conversion_map_resnet:
+                v = v.replace(hf_part, sd_part)
+            mapping[k] = v
+    for k, v in mapping.items():
+        for sd_part, hf_part in unet_conversion_map_layer:
+            v = v.replace(hf_part, sd_part)
+        mapping[k] = v
+    new_state_dict = {v: unet_state_dict[k] for k, v in mapping.items()}
+
+    if v2:
+        conv_transformer_to_linear(new_state_dict)
+
+    return new_state_dict
+
+def controlnet_conversion_map():
+    unet_conversion_map = [
+        ("time_embed.0.weight", "time_embedding.linear_1.weight"),
+        ("time_embed.0.bias", "time_embedding.linear_1.bias"),
+        ("time_embed.2.weight", "time_embedding.linear_2.weight"),
+        ("time_embed.2.bias", "time_embedding.linear_2.bias"),
+        ("input_blocks.0.0.weight", "conv_in.weight"),
+        ("input_blocks.0.0.bias", "conv_in.bias"),
+        ("middle_block_out.0.weight", "controlnet_mid_block.weight"),
+        ("middle_block_out.0.bias", "controlnet_mid_block.bias"),
+    ]
+
+    unet_conversion_map_resnet = [
+        ("in_layers.0", "norm1"),
+        ("in_layers.2", "conv1"),
+        ("out_layers.0", "norm2"),
+        ("out_layers.3", "conv2"),
+        ("emb_layers.1", "time_emb_proj"),
+        ("skip_connection", "conv_shortcut"),
+    ]
+
+    unet_conversion_map_layer = []
+    for i in range(4):
+        for j in range(2):
+            hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}."
+            sd_down_res_prefix = f"input_blocks.{3*i + j + 1}.0."
+            unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix))
+
+            if i < 3:
+                hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}."
+                sd_down_atn_prefix = f"input_blocks.{3*i + j + 1}.1."
+                unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix))
+
+        if i < 3:
+            hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv."
+            sd_downsample_prefix = f"input_blocks.{3*(i+1)}.0.op."
+            unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix))
+
+    hf_mid_atn_prefix = "mid_block.attentions.0."
+    sd_mid_atn_prefix = "middle_block.1."
+    unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix))
+
+    for j in range(2):
+        hf_mid_res_prefix = f"mid_block.resnets.{j}."
+        sd_mid_res_prefix = f"middle_block.{2*j}."
+        unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix))
+
+    controlnet_cond_embedding_names = (
+        ["conv_in"] + [f"blocks.{i}" for i in range(6)] + ["conv_out"]
+    )
+    for i, hf_prefix in enumerate(controlnet_cond_embedding_names):
+        hf_prefix = f"controlnet_cond_embedding.{hf_prefix}."
+        sd_prefix = f"input_hint_block.{i*2}."
+        unet_conversion_map_layer.append((sd_prefix, hf_prefix))
+
+    for i in range(12):
+        hf_prefix = f"controlnet_down_blocks.{i}."
+        sd_prefix = f"zero_convs.{i}.0."
+        unet_conversion_map_layer.append((sd_prefix, hf_prefix))
+
+    return unet_conversion_map, unet_conversion_map_resnet, unet_conversion_map_layer
+
+
+def convert_controlnet_state_dict_to_sd(controlnet_state_dict):
+    unet_conversion_map, unet_conversion_map_resnet, unet_conversion_map_layer = controlnet_conversion_map()
+
+    mapping = {k: k for k in controlnet_state_dict.keys()}
+    for sd_name, diffusers_name in unet_conversion_map:
+        mapping[diffusers_name] = sd_name
+    for k, v in mapping.items():
+        if "resnets" in k:
+            for sd_part, diffusers_part in unet_conversion_map_resnet:
+                v = v.replace(diffusers_part, sd_part)
+            mapping[k] = v
+    for k, v in mapping.items():
+        for sd_part, diffusers_part in unet_conversion_map_layer:
+            v = v.replace(diffusers_part, sd_part)
+        mapping[k] = v
+    new_state_dict = {v: controlnet_state_dict[k] for k, v in mapping.items()}
+    return new_state_dict
+
+def convert_controlnet_state_dict_to_diffusers(controlnet_state_dict):
+    unet_conversion_map, unet_conversion_map_resnet, unet_conversion_map_layer = controlnet_conversion_map()
+
+    mapping = {k: k for k in controlnet_state_dict.keys()}
+    for sd_name, diffusers_name in unet_conversion_map:
+        mapping[sd_name] = diffusers_name
+    for k, v in mapping.items():
+        for sd_part, diffusers_part in unet_conversion_map_layer:
+            v = v.replace(sd_part, diffusers_part)
+        mapping[k] = v
+    for k, v in mapping.items():
+        if "resnets" in v:
+            for sd_part, diffusers_part in unet_conversion_map_resnet:
+                v = v.replace(sd_part, diffusers_part)
+            mapping[k] = v
+    new_state_dict = {v: controlnet_state_dict[k] for k, v in mapping.items()}
+    return new_state_dict
+
+# ================#
+# VAE Conversion #
+# ================#
+
+
+def reshape_weight_for_sd(w):
+    # convert HF linear weights to SD conv2d weights
+    return w.reshape(*w.shape, 1, 1)
+
+
+def convert_vae_state_dict(vae_state_dict):
+    vae_conversion_map = [
+        # (stable-diffusion, HF Diffusers)
+        ("nin_shortcut", "conv_shortcut"),
+        ("norm_out", "conv_norm_out"),
+        ("mid.attn_1.", "mid_block.attentions.0."),
+    ]
+
+    for i in range(4):
+        # down_blocks have two resnets
+        for j in range(2):
+            hf_down_prefix = f"encoder.down_blocks.{i}.resnets.{j}."
+            sd_down_prefix = f"encoder.down.{i}.block.{j}."
+            vae_conversion_map.append((sd_down_prefix, hf_down_prefix))
+
+        if i < 3:
+            hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0."
+            sd_downsample_prefix = f"down.{i}.downsample."
+            vae_conversion_map.append((sd_downsample_prefix, hf_downsample_prefix))
+
+            hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0."
+            sd_upsample_prefix = f"up.{3-i}.upsample."
+            vae_conversion_map.append((sd_upsample_prefix, hf_upsample_prefix))
+
+        # up_blocks have three resnets
+        # also, up blocks in hf are numbered in reverse from sd
+        for j in range(3):
+            hf_up_prefix = f"decoder.up_blocks.{i}.resnets.{j}."
+            sd_up_prefix = f"decoder.up.{3-i}.block.{j}."
+            vae_conversion_map.append((sd_up_prefix, hf_up_prefix))
+
+    # this part accounts for mid blocks in both the encoder and the decoder
+    for i in range(2):
+        hf_mid_res_prefix = f"mid_block.resnets.{i}."
+        sd_mid_res_prefix = f"mid.block_{i+1}."
+        vae_conversion_map.append((sd_mid_res_prefix, hf_mid_res_prefix))
+
+    if diffusers.__version__ < "0.17.0":
+        vae_conversion_map_attn = [
+            # (stable-diffusion, HF Diffusers)
+            ("norm.", "group_norm."),
+            ("q.", "query."),
+            ("k.", "key."),
+            ("v.", "value."),
+            ("proj_out.", "proj_attn."),
+        ]
+    else:
+        vae_conversion_map_attn = [
+            # (stable-diffusion, HF Diffusers)
+            ("norm.", "group_norm."),
+            ("q.", "to_q."),
+            ("k.", "to_k."),
+            ("v.", "to_v."),
+            ("proj_out.", "to_out.0."),
+        ]
+
+    mapping = {k: k for k in vae_state_dict.keys()}
+    for k, v in mapping.items():
+        for sd_part, hf_part in vae_conversion_map:
+            v = v.replace(hf_part, sd_part)
+        mapping[k] = v
+    for k, v in mapping.items():
+        if "attentions" in k:
+            for sd_part, hf_part in vae_conversion_map_attn:
+                v = v.replace(hf_part, sd_part)
+            mapping[k] = v
+    new_state_dict = {v: vae_state_dict[k] for k, v in mapping.items()}
+    weights_to_convert = ["q", "k", "v", "proj_out"]
+    for k, v in new_state_dict.items():
+        for weight_name in weights_to_convert:
+            if f"mid.attn_1.{weight_name}.weight" in k:
+                # print(f"Reshaping {k} for SD format: shape {v.shape} -> {v.shape} x 1 x 1")
+                new_state_dict[k] = reshape_weight_for_sd(v)
+
+    return new_state_dict
+
+
+# endregion
+
+# region 自作のモデル読み書きなど
+
+
+def is_safetensors(path):
+    return os.path.splitext(path)[1].lower() == ".safetensors"
+
+
+def load_checkpoint_with_text_encoder_conversion(ckpt_path, device="cpu"):
+    # text encoderの格納形式が違うモデルに対応する ('text_model'がない)
+    TEXT_ENCODER_KEY_REPLACEMENTS = [
+        ("cond_stage_model.transformer.embeddings.", "cond_stage_model.transformer.text_model.embeddings."),
+        ("cond_stage_model.transformer.encoder.", "cond_stage_model.transformer.text_model.encoder."),
+        ("cond_stage_model.transformer.final_layer_norm.", "cond_stage_model.transformer.text_model.final_layer_norm."),
+    ]
+
+    if is_safetensors(ckpt_path):
+        checkpoint = None
+        state_dict = load_file(ckpt_path)  # , device) # may causes error
+    else:
+        checkpoint = torch.load(ckpt_path, map_location=device)
+        if "state_dict" in checkpoint:
+            state_dict = checkpoint["state_dict"]
+        else:
+            state_dict = checkpoint
+            checkpoint = None
+
+    key_reps = []
+    for rep_from, rep_to in TEXT_ENCODER_KEY_REPLACEMENTS:
+        for key in state_dict.keys():
+            if key.startswith(rep_from):
+                new_key = rep_to + key[len(rep_from) :]
+                key_reps.append((key, new_key))
+
+    for key, new_key in key_reps:
+        state_dict[new_key] = state_dict[key]
+        del state_dict[key]
+
+    return checkpoint, state_dict
+
+
+# TODO dtype指定の動作が怪しいので確認する text_encoderを指定形式で作れるか未確認
+def load_models_from_stable_diffusion_checkpoint(v2, ckpt_path, device="cpu", dtype=None, unet_use_linear_projection_in_v2=True):
+    _, state_dict = load_checkpoint_with_text_encoder_conversion(ckpt_path, device)
+
+    # Convert the UNet2DConditionModel model.
+    unet_config = create_unet_diffusers_config(v2, unet_use_linear_projection_in_v2)
+    converted_unet_checkpoint = convert_ldm_unet_checkpoint(v2, state_dict, unet_config)
+
+    unet = UNet2DConditionModel(**unet_config).to(device)
+    info = unet.load_state_dict(converted_unet_checkpoint)
+    print("loading u-net:", info)
+
+    # Convert the VAE model.
+    vae_config = create_vae_diffusers_config()
+    converted_vae_checkpoint = convert_ldm_vae_checkpoint(state_dict, vae_config)
+
+    vae = AutoencoderKL(**vae_config).to(device)
+    info = vae.load_state_dict(converted_vae_checkpoint)
+    print("loading vae:", info)
+
+    # convert text_model
+    if v2:
+        converted_text_encoder_checkpoint = convert_ldm_clip_checkpoint_v2(state_dict, 77)
+        cfg = CLIPTextConfig(
+            vocab_size=49408,
+            hidden_size=1024,
+            intermediate_size=4096,
+            num_hidden_layers=23,
+            num_attention_heads=16,
+            max_position_embeddings=77,
+            hidden_act="gelu",
+            layer_norm_eps=1e-05,
+            dropout=0.0,
+            attention_dropout=0.0,
+            initializer_range=0.02,
+            initializer_factor=1.0,
+            pad_token_id=1,
+            bos_token_id=0,
+            eos_token_id=2,
+            model_type="clip_text_model",
+            projection_dim=512,
+            torch_dtype="float32",
+            transformers_version="4.25.0.dev0",
+        )
+        text_model = CLIPTextModel._from_config(cfg)
+        info = text_model.load_state_dict(converted_text_encoder_checkpoint)
+    else:
+        converted_text_encoder_checkpoint = convert_ldm_clip_checkpoint_v1(state_dict)
+
+        # logging.set_verbosity_error()  # don't show annoying warning
+        # text_model = CLIPTextModel.from_pretrained("openai/clip-vit-large-patch14").to(device)
+        # logging.set_verbosity_warning()
+        # print(f"config: {text_model.config}")
+        cfg = CLIPTextConfig(
+            vocab_size=49408,
+            hidden_size=768,
+            intermediate_size=3072,
+            num_hidden_layers=12,
+            num_attention_heads=12,
+            max_position_embeddings=77,
+            hidden_act="quick_gelu",
+            layer_norm_eps=1e-05,
+            dropout=0.0,
+            attention_dropout=0.0,
+            initializer_range=0.02,
+            initializer_factor=1.0,
+            pad_token_id=1,
+            bos_token_id=0,
+            eos_token_id=2,
+            model_type="clip_text_model",
+            projection_dim=768,
+            torch_dtype="float32",
+        )
+        text_model = CLIPTextModel._from_config(cfg)
+        info = text_model.load_state_dict(converted_text_encoder_checkpoint)
+    print("loading text encoder:", info)
+
+    return text_model, vae, unet
+
+def get_model_version_str_for_sd1_sd2(v2, v_parameterization):
+    # only for reference
+    version_str = "sd"
+    if v2:
+        version_str += "_v2"
+    else:
+        version_str += "_v1"
+    if v_parameterization:
+        version_str += "_v"
+    return version_str
+
+def convert_text_encoder_state_dict_to_sd_v2(checkpoint, make_dummy_weights=False):
+    def convert_key(key):
+        # position_idsの除去
+        if ".position_ids" in key:
+            return None
+
+        # common
+        key = key.replace("text_model.encoder.", "transformer.")
+        key = key.replace("text_model.", "")
+        if "layers" in key:
+            # resblocks conversion
+            key = key.replace(".layers.", ".resblocks.")
+            if ".layer_norm" in key:
+                key = key.replace(".layer_norm", ".ln_")
+            elif ".mlp." in key:
+                key = key.replace(".fc1.", ".c_fc.")
+                key = key.replace(".fc2.", ".c_proj.")
+            elif ".self_attn.out_proj" in key:
+                key = key.replace(".self_attn.out_proj.", ".attn.out_proj.")
+            elif ".self_attn." in key:
+                key = None  # 特殊なので後で処理する
+            else:
+                raise ValueError(f"unexpected key in DiffUsers model: {key}")
+        elif ".position_embedding" in key:
+            key = key.replace("embeddings.position_embedding.weight", "positional_embedding")
+        elif ".token_embedding" in key:
+            key = key.replace("embeddings.token_embedding.weight", "token_embedding.weight")
+        elif "final_layer_norm" in key:
+            key = key.replace("final_layer_norm", "ln_final")
+        return key
+
+    keys = list(checkpoint.keys())
+    new_sd = {}
+    for key in keys:
+        new_key = convert_key(key)
+        if new_key is None:
+            continue
+        new_sd[new_key] = checkpoint[key]
+
+    # attnの変換
+    for key in keys:
+        if "layers" in key and "q_proj" in key:
+            # 三つを結合
+            key_q = key
+            key_k = key.replace("q_proj", "k_proj")
+            key_v = key.replace("q_proj", "v_proj")
+
+            value_q = checkpoint[key_q]
+            value_k = checkpoint[key_k]
+            value_v = checkpoint[key_v]
+            value = torch.cat([value_q, value_k, value_v])
+
+            new_key = key.replace("text_model.encoder.layers.", "transformer.resblocks.")
+            new_key = new_key.replace(".self_attn.q_proj.", ".attn.in_proj_")
+            new_sd[new_key] = value
+
+    # 最後の層などを捏造するか
+    if make_dummy_weights:
+        print("make dummy weights for resblock.23, text_projection and logit scale.")
+        keys = list(new_sd.keys())
+        for key in keys:
+            if key.startswith("transformer.resblocks.22."):
+                new_sd[key.replace(".22.", ".23.")] = new_sd[key].clone()  # copyしないとsafetensorsの保存で落ちる
+
+        # Diffusersに含まれない重みを作っておく
+        new_sd["text_projection"] = torch.ones((1024, 1024), dtype=new_sd[keys[0]].dtype, device=new_sd[keys[0]].device)
+        new_sd["logit_scale"] = torch.tensor(1)
+
+    return new_sd
+
+
+def save_stable_diffusion_checkpoint(v2, output_file, text_encoder, unet, ckpt_path, epochs, steps, save_dtype=None, vae=None):
+    if ckpt_path is not None:
+        # epoch/stepを参照する。またVAEがメモリ上にないときなど、もう一度VAEを含めて読み込む
+        checkpoint, state_dict = load_checkpoint_with_text_encoder_conversion(ckpt_path)
+        if checkpoint is None:  # safetensors または state_dictのckpt
+            checkpoint = {}
+            strict = False
+        else:
+            strict = True
+        if "state_dict" in state_dict:
+            del state_dict["state_dict"]
+    else:
+        # 新しく作る
+        assert vae is not None, "VAE is required to save a checkpoint without a given checkpoint"
+        checkpoint = {}
+        state_dict = {}
+        strict = False
+
+    def update_sd(prefix, sd):
+        for k, v in sd.items():
+            key = prefix + k
+            assert not strict or key in state_dict, f"Illegal key in save SD: {key}"
+            if save_dtype is not None:
+                v = v.detach().clone().to("cpu").to(save_dtype)
+            state_dict[key] = v
+
+    # Convert the UNet model
+    unet_state_dict = convert_unet_state_dict_to_sd(v2, unet.state_dict())
+    update_sd("model.diffusion_model.", unet_state_dict)
+
+    # Convert the text encoder model
+    if v2:
+        make_dummy = ckpt_path is None  # 参照元のcheckpointがない場合は最後の層を前の層から複製して作るなどダミーの重みを入れる
+        text_enc_dict = convert_text_encoder_state_dict_to_sd_v2(text_encoder.state_dict(), make_dummy)
+        update_sd("cond_stage_model.model.", text_enc_dict)
+    else:
+        text_enc_dict = text_encoder.state_dict()
+        update_sd("cond_stage_model.transformer.", text_enc_dict)
+
+    # Convert the VAE
+    if vae is not None:
+        vae_dict = convert_vae_state_dict(vae.state_dict())
+        update_sd("first_stage_model.", vae_dict)
+
+    # Put together new checkpoint
+    key_count = len(state_dict.keys())
+    new_ckpt = {"state_dict": state_dict}
+
+    # epoch and global_step are sometimes not int
+    try:
+        if "epoch" in checkpoint:
+            epochs += checkpoint["epoch"]
+        if "global_step" in checkpoint:
+            steps += checkpoint["global_step"]
+    except:
+        pass
+
+    new_ckpt["epoch"] = epochs
+    new_ckpt["global_step"] = steps
+
+    if is_safetensors(output_file):
+        # TODO Tensor以外のdictの値を削除したほうがいいか
+        save_file(state_dict, output_file)
+    else:
+        torch.save(new_ckpt, output_file)
+
+    return key_count
+
+
+def save_diffusers_checkpoint(v2, output_dir, text_encoder, unet, pretrained_model_name_or_path, vae=None, use_safetensors=False):
+    if pretrained_model_name_or_path is None:
+        # load default settings for v1/v2
+        if v2:
+            pretrained_model_name_or_path = DIFFUSERS_REF_MODEL_ID_V2
+        else:
+            pretrained_model_name_or_path = DIFFUSERS_REF_MODEL_ID_V1
+
+    scheduler = DDIMScheduler.from_pretrained(pretrained_model_name_or_path, subfolder="scheduler")
+    tokenizer = CLIPTokenizer.from_pretrained(pretrained_model_name_or_path, subfolder="tokenizer")
+    if vae is None:
+        vae = AutoencoderKL.from_pretrained(pretrained_model_name_or_path, subfolder="vae")
+
+    pipeline = StableDiffusionPipeline(
+        unet=unet,
+        text_encoder=text_encoder,
+        vae=vae,
+        scheduler=scheduler,
+        tokenizer=tokenizer,
+        safety_checker=None,
+        feature_extractor=None,
+        requires_safety_checker=None,
+    )
+    pipeline.save_pretrained(output_dir, safe_serialization=use_safetensors)
+
+
+VAE_PREFIX = "first_stage_model."
+
+
+def load_vae(vae_id, dtype):
+    print(f"load VAE: {vae_id}")
+    if os.path.isdir(vae_id) or not os.path.isfile(vae_id):
+        # Diffusers local/remote
+        try:
+            vae = AutoencoderKL.from_pretrained(vae_id, subfolder=None, torch_dtype=dtype)
+        except EnvironmentError as e:
+            print(f"exception occurs in loading vae: {e}")
+            print("retry with subfolder='vae'")
+            vae = AutoencoderKL.from_pretrained(vae_id, subfolder="vae", torch_dtype=dtype)
+        return vae
+
+    # local
+    vae_config = create_vae_diffusers_config()
+
+    if vae_id.endswith(".bin"):
+        # SD 1.5 VAE on Huggingface
+        converted_vae_checkpoint = torch.load(vae_id, map_location="cpu")
+    else:
+        # StableDiffusion
+        vae_model = load_file(vae_id, "cpu") if is_safetensors(vae_id) else torch.load(vae_id, map_location="cpu")
+        vae_sd = vae_model["state_dict"] if "state_dict" in vae_model else vae_model
+
+        # vae only or full model
+        full_model = False
+        for vae_key in vae_sd:
+            if vae_key.startswith(VAE_PREFIX):
+                full_model = True
+                break
+        if not full_model:
+            sd = {}
+            for key, value in vae_sd.items():
+                sd[VAE_PREFIX + key] = value
+            vae_sd = sd
+            del sd
+
+        # Convert the VAE model.
+        converted_vae_checkpoint = convert_ldm_vae_checkpoint(vae_sd, vae_config)
+
+    vae = AutoencoderKL(**vae_config)
+    vae.load_state_dict(converted_vae_checkpoint)
+    return vae
+
+
+# endregion
+
+
+def make_bucket_resolutions(max_reso, min_size=256, max_size=1024, divisible=64):
+    max_width, max_height = max_reso
+    max_area = (max_width // divisible) * (max_height // divisible)
+
+    resos = set()
+
+    size = int(math.sqrt(max_area)) * divisible
+    resos.add((size, size))
+
+    size = min_size
+    while size <= max_size:
+        width = size
+        height = min(max_size, (max_area // (width // divisible)) * divisible)
+        resos.add((width, height))
+        resos.add((height, width))
+
+        # # make additional resos
+        # if width >= height and width - divisible >= min_size:
+        #   resos.add((width - divisible, height))
+        #   resos.add((height, width - divisible))
+        # if height >= width and height - divisible >= min_size:
+        #   resos.add((width, height - divisible))
+        #   resos.add((height - divisible, width))
+
+        size += divisible
+
+    resos = list(resos)
+    resos.sort()
+    return resos
+
+
+if __name__ == "__main__":
+    resos = make_bucket_resolutions((512, 768))
+    print(len(resos))
+    print(resos)
+    aspect_ratios = [w / h for w, h in resos]
+    print(aspect_ratios)
+
+    ars = set()
+    for ar in aspect_ratios:
+        if ar in ars:
+            print("error! duplicate ar:", ar)
+        ars.add(ar)
diff --git a/library/original_unet.py b/library/original_unet.py
new file mode 100644
index 0000000000000000000000000000000000000000..c0028ddc2a0fda68d191a513620fd0b72e36a260
--- /dev/null
+++ b/library/original_unet.py
@@ -0,0 +1,1606 @@
+# Diffusers 0.10.2からStable Diffusionに必要な部分だけを持ってくる
+# 条件分岐等で不要な部分は削除している
+# コードの多くはDiffusersからコピーしている
+# 制約として、モデルのstate_dictがDiffusers 0.10.2のものと同じ形式である必要がある
+
+# Copy from Diffusers 0.10.2 for Stable Diffusion. Most of the code is copied from Diffusers.
+# Unnecessary parts are deleted by condition branching.
+# As a constraint, the state_dict of the model must be in the same format as that of Diffusers 0.10.2
+
+"""
+v1.5とv2.1の相違点は
+- attention_head_dimがintかlist[int]か
+- cross_attention_dimが768か1024か
+- use_linear_projection: trueがない（=False, 1.5）かあるか
+- upcast_attentionがFalse(1.5)かTrue(2.1)か
+- （以下は多分無視していい）
+- sample_sizeが64か96か
+- dual_cross_attentionがあるかないか
+- num_class_embedsがあるかないか
+- only_cross_attentionがあるかないか
+
+v1.5
+{
+  "_class_name": "UNet2DConditionModel",
+  "_diffusers_version": "0.6.0",
+  "act_fn": "silu",
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "out_channels": 4,
+  "sample_size": 64,
+  "up_block_types": [
+    "UpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D"
+  ]
+}
+
+v2.1
+{
+  "_class_name": "UNet2DConditionModel",
+  "_diffusers_version": "0.10.0.dev0",
+  "act_fn": "silu",
+  "attention_head_dim": [
+    5,
+    10,
+    20,
+    20
+  ],
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "cross_attention_dim": 1024,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 4,
+  "sample_size": 96,
+  "up_block_types": [
+    "UpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D"
+  ],
+  "use_linear_projection": true,
+  "upcast_attention": true
+}
+"""
+
+import math
+from types import SimpleNamespace
+from typing import Dict, Optional, Tuple, Union
+import torch
+from torch import nn
+from torch.nn import functional as F
+from einops import rearrange
+
+BLOCK_OUT_CHANNELS: Tuple[int] = (320, 640, 1280, 1280)
+TIMESTEP_INPUT_DIM = BLOCK_OUT_CHANNELS[0]
+TIME_EMBED_DIM = BLOCK_OUT_CHANNELS[0] * 4
+IN_CHANNELS: int = 4
+OUT_CHANNELS: int = 4
+LAYERS_PER_BLOCK: int = 2
+LAYERS_PER_BLOCK_UP: int = LAYERS_PER_BLOCK + 1
+TIME_EMBED_FLIP_SIN_TO_COS: bool = True
+TIME_EMBED_FREQ_SHIFT: int = 0
+NORM_GROUPS: int = 32
+NORM_EPS: float = 1e-5
+TRANSFORMER_NORM_NUM_GROUPS = 32
+
+DOWN_BLOCK_TYPES = ["CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D"]
+UP_BLOCK_TYPES = ["UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"]
+
+
+# region memory effcient attention
+
+# FlashAttentionを使うCrossAttention
+# based on https://github.com/lucidrains/memory-efficient-attention-pytorch/blob/main/memory_efficient_attention_pytorch/flash_attention.py
+# LICENSE MIT https://github.com/lucidrains/memory-efficient-attention-pytorch/blob/main/LICENSE
+
+# constants
+
+EPSILON = 1e-6
+
+# helper functions
+
+
+def exists(val):
+    return val is not None
+
+
+def default(val, d):
+    return val if exists(val) else d
+
+
+# flash attention forwards and backwards
+
+# https://arxiv.org/abs/2205.14135
+
+
+class FlashAttentionFunction(torch.autograd.Function):
+    @staticmethod
+    @torch.no_grad()
+    def forward(ctx, q, k, v, mask, causal, q_bucket_size, k_bucket_size):
+        """Algorithm 2 in the paper"""
+
+        device = q.device
+        dtype = q.dtype
+        max_neg_value = -torch.finfo(q.dtype).max
+        qk_len_diff = max(k.shape[-2] - q.shape[-2], 0)
+
+        o = torch.zeros_like(q)
+        all_row_sums = torch.zeros((*q.shape[:-1], 1), dtype=dtype, device=device)
+        all_row_maxes = torch.full((*q.shape[:-1], 1), max_neg_value, dtype=dtype, device=device)
+
+        scale = q.shape[-1] ** -0.5
+
+        if not exists(mask):
+            mask = (None,) * math.ceil(q.shape[-2] / q_bucket_size)
+        else:
+            mask = rearrange(mask, "b n -> b 1 1 n")
+            mask = mask.split(q_bucket_size, dim=-1)
+
+        row_splits = zip(
+            q.split(q_bucket_size, dim=-2),
+            o.split(q_bucket_size, dim=-2),
+            mask,
+            all_row_sums.split(q_bucket_size, dim=-2),
+            all_row_maxes.split(q_bucket_size, dim=-2),
+        )
+
+        for ind, (qc, oc, row_mask, row_sums, row_maxes) in enumerate(row_splits):
+            q_start_index = ind * q_bucket_size - qk_len_diff
+
+            col_splits = zip(
+                k.split(k_bucket_size, dim=-2),
+                v.split(k_bucket_size, dim=-2),
+            )
+
+            for k_ind, (kc, vc) in enumerate(col_splits):
+                k_start_index = k_ind * k_bucket_size
+
+                attn_weights = torch.einsum("... i d, ... j d -> ... i j", qc, kc) * scale
+
+                if exists(row_mask):
+                    attn_weights.masked_fill_(~row_mask, max_neg_value)
+
+                if causal and q_start_index < (k_start_index + k_bucket_size - 1):
+                    causal_mask = torch.ones((qc.shape[-2], kc.shape[-2]), dtype=torch.bool, device=device).triu(
+                        q_start_index - k_start_index + 1
+                    )
+                    attn_weights.masked_fill_(causal_mask, max_neg_value)
+
+                block_row_maxes = attn_weights.amax(dim=-1, keepdims=True)
+                attn_weights -= block_row_maxes
+                exp_weights = torch.exp(attn_weights)
+
+                if exists(row_mask):
+                    exp_weights.masked_fill_(~row_mask, 0.0)
+
+                block_row_sums = exp_weights.sum(dim=-1, keepdims=True).clamp(min=EPSILON)
+
+                new_row_maxes = torch.maximum(block_row_maxes, row_maxes)
+
+                exp_values = torch.einsum("... i j, ... j d -> ... i d", exp_weights, vc)
+
+                exp_row_max_diff = torch.exp(row_maxes - new_row_maxes)
+                exp_block_row_max_diff = torch.exp(block_row_maxes - new_row_maxes)
+
+                new_row_sums = exp_row_max_diff * row_sums + exp_block_row_max_diff * block_row_sums
+
+                oc.mul_((row_sums / new_row_sums) * exp_row_max_diff).add_((exp_block_row_max_diff / new_row_sums) * exp_values)
+
+                row_maxes.copy_(new_row_maxes)
+                row_sums.copy_(new_row_sums)
+
+        ctx.args = (causal, scale, mask, q_bucket_size, k_bucket_size)
+        ctx.save_for_backward(q, k, v, o, all_row_sums, all_row_maxes)
+
+        return o
+
+    @staticmethod
+    @torch.no_grad()
+    def backward(ctx, do):
+        """Algorithm 4 in the paper"""
+
+        causal, scale, mask, q_bucket_size, k_bucket_size = ctx.args
+        q, k, v, o, l, m = ctx.saved_tensors
+
+        device = q.device
+
+        max_neg_value = -torch.finfo(q.dtype).max
+        qk_len_diff = max(k.shape[-2] - q.shape[-2], 0)
+
+        dq = torch.zeros_like(q)
+        dk = torch.zeros_like(k)
+        dv = torch.zeros_like(v)
+
+        row_splits = zip(
+            q.split(q_bucket_size, dim=-2),
+            o.split(q_bucket_size, dim=-2),
+            do.split(q_bucket_size, dim=-2),
+            mask,
+            l.split(q_bucket_size, dim=-2),
+            m.split(q_bucket_size, dim=-2),
+            dq.split(q_bucket_size, dim=-2),
+        )
+
+        for ind, (qc, oc, doc, row_mask, lc, mc, dqc) in enumerate(row_splits):
+            q_start_index = ind * q_bucket_size - qk_len_diff
+
+            col_splits = zip(
+                k.split(k_bucket_size, dim=-2),
+                v.split(k_bucket_size, dim=-2),
+                dk.split(k_bucket_size, dim=-2),
+                dv.split(k_bucket_size, dim=-2),
+            )
+
+            for k_ind, (kc, vc, dkc, dvc) in enumerate(col_splits):
+                k_start_index = k_ind * k_bucket_size
+
+                attn_weights = torch.einsum("... i d, ... j d -> ... i j", qc, kc) * scale
+
+                if causal and q_start_index < (k_start_index + k_bucket_size - 1):
+                    causal_mask = torch.ones((qc.shape[-2], kc.shape[-2]), dtype=torch.bool, device=device).triu(
+                        q_start_index - k_start_index + 1
+                    )
+                    attn_weights.masked_fill_(causal_mask, max_neg_value)
+
+                exp_attn_weights = torch.exp(attn_weights - mc)
+
+                if exists(row_mask):
+                    exp_attn_weights.masked_fill_(~row_mask, 0.0)
+
+                p = exp_attn_weights / lc
+
+                dv_chunk = torch.einsum("... i j, ... i d -> ... j d", p, doc)
+                dp = torch.einsum("... i d, ... j d -> ... i j", doc, vc)
+
+                D = (doc * oc).sum(dim=-1, keepdims=True)
+                ds = p * scale * (dp - D)
+
+                dq_chunk = torch.einsum("... i j, ... j d -> ... i d", ds, kc)
+                dk_chunk = torch.einsum("... i j, ... i d -> ... j d", ds, qc)
+
+                dqc.add_(dq_chunk)
+                dkc.add_(dk_chunk)
+                dvc.add_(dv_chunk)
+
+        return dq, dk, dv, None, None, None, None
+
+
+# endregion
+
+
+def get_parameter_dtype(parameter: torch.nn.Module):
+    return next(parameter.parameters()).dtype
+
+
+def get_parameter_device(parameter: torch.nn.Module):
+    return next(parameter.parameters()).device
+
+
+def get_timestep_embedding(
+    timesteps: torch.Tensor,
+    embedding_dim: int,
+    flip_sin_to_cos: bool = False,
+    downscale_freq_shift: float = 1,
+    scale: float = 1,
+    max_period: int = 10000,
+):
+    """
+    This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings.
+
+    :param timesteps: a 1-D Tensor of N indices, one per batch element.
+                      These may be fractional.
+    :param embedding_dim: the dimension of the output. :param max_period: controls the minimum frequency of the
+    embeddings. :return: an [N x dim] Tensor of positional embeddings.
+    """
+    assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"
+
+    half_dim = embedding_dim // 2
+    exponent = -math.log(max_period) * torch.arange(start=0, end=half_dim, dtype=torch.float32, device=timesteps.device)
+    exponent = exponent / (half_dim - downscale_freq_shift)
+
+    emb = torch.exp(exponent)
+    emb = timesteps[:, None].float() * emb[None, :]
+
+    # scale embeddings
+    emb = scale * emb
+
+    # concat sine and cosine embeddings
+    emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1)
+
+    # flip sine and cosine embeddings
+    if flip_sin_to_cos:
+        emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)
+
+    # zero pad
+    if embedding_dim % 2 == 1:
+        emb = torch.nn.functional.pad(emb, (0, 1, 0, 0))
+    return emb
+
+
+class SampleOutput:
+    def __init__(self, sample):
+        self.sample = sample
+
+
+class TimestepEmbedding(nn.Module):
+    def __init__(self, in_channels: int, time_embed_dim: int, act_fn: str = "silu", out_dim: int = None):
+        super().__init__()
+
+        self.linear_1 = nn.Linear(in_channels, time_embed_dim)
+        self.act = None
+        if act_fn == "silu":
+            self.act = nn.SiLU()
+        elif act_fn == "mish":
+            self.act = nn.Mish()
+
+        if out_dim is not None:
+            time_embed_dim_out = out_dim
+        else:
+            time_embed_dim_out = time_embed_dim
+        self.linear_2 = nn.Linear(time_embed_dim, time_embed_dim_out)
+
+    def forward(self, sample):
+        sample = self.linear_1(sample)
+
+        if self.act is not None:
+            sample = self.act(sample)
+
+        sample = self.linear_2(sample)
+        return sample
+
+
+class Timesteps(nn.Module):
+    def __init__(self, num_channels: int, flip_sin_to_cos: bool, downscale_freq_shift: float):
+        super().__init__()
+        self.num_channels = num_channels
+        self.flip_sin_to_cos = flip_sin_to_cos
+        self.downscale_freq_shift = downscale_freq_shift
+
+    def forward(self, timesteps):
+        t_emb = get_timestep_embedding(
+            timesteps,
+            self.num_channels,
+            flip_sin_to_cos=self.flip_sin_to_cos,
+            downscale_freq_shift=self.downscale_freq_shift,
+        )
+        return t_emb
+
+
+class ResnetBlock2D(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+
+        self.norm1 = torch.nn.GroupNorm(num_groups=NORM_GROUPS, num_channels=in_channels, eps=NORM_EPS, affine=True)
+
+        self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+
+        self.time_emb_proj = torch.nn.Linear(TIME_EMBED_DIM, out_channels)
+
+        self.norm2 = torch.nn.GroupNorm(num_groups=NORM_GROUPS, num_channels=out_channels, eps=NORM_EPS, affine=True)
+        self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+
+        # if non_linearity == "swish":
+        self.nonlinearity = lambda x: F.silu(x)
+
+        self.use_in_shortcut = self.in_channels != self.out_channels
+
+        self.conv_shortcut = None
+        if self.use_in_shortcut:
+            self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+    def forward(self, input_tensor, temb):
+        hidden_states = input_tensor
+
+        hidden_states = self.norm1(hidden_states)
+        hidden_states = self.nonlinearity(hidden_states)
+
+        hidden_states = self.conv1(hidden_states)
+
+        temb = self.time_emb_proj(self.nonlinearity(temb))[:, :, None, None]
+        hidden_states = hidden_states + temb
+
+        hidden_states = self.norm2(hidden_states)
+        hidden_states = self.nonlinearity(hidden_states)
+
+        hidden_states = self.conv2(hidden_states)
+
+        if self.conv_shortcut is not None:
+            input_tensor = self.conv_shortcut(input_tensor)
+
+        output_tensor = input_tensor + hidden_states
+
+        return output_tensor
+
+
+class DownBlock2D(nn.Module):
+    def __init__(
+        self,
+        in_channels: int,
+        out_channels: int,
+        add_downsample=True,
+    ):
+        super().__init__()
+
+        self.has_cross_attention = False
+        resnets = []
+
+        for i in range(LAYERS_PER_BLOCK):
+            in_channels = in_channels if i == 0 else out_channels
+            resnets.append(
+                ResnetBlock2D(
+                    in_channels=in_channels,
+                    out_channels=out_channels,
+                )
+            )
+        self.resnets = nn.ModuleList(resnets)
+
+        if add_downsample:
+            self.downsamplers = [Downsample2D(out_channels, out_channels=out_channels)]
+        else:
+            self.downsamplers = None
+
+        self.gradient_checkpointing = False
+
+    def set_use_memory_efficient_attention(self, xformers, mem_eff):
+        pass
+
+    def set_use_sdpa(self, sdpa):
+        pass
+
+    def forward(self, hidden_states, temb=None):
+        output_states = ()
+
+        for resnet in self.resnets:
+            if self.training and self.gradient_checkpointing:
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        return module(*inputs)
+
+                    return custom_forward
+
+                hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
+            else:
+                hidden_states = resnet(hidden_states, temb)
+
+            output_states += (hidden_states,)
+
+        if self.downsamplers is not None:
+            for downsampler in self.downsamplers:
+                hidden_states = downsampler(hidden_states)
+
+            output_states += (hidden_states,)
+
+        return hidden_states, output_states
+
+
+class Downsample2D(nn.Module):
+    def __init__(self, channels, out_channels):
+        super().__init__()
+
+        self.channels = channels
+        self.out_channels = out_channels
+
+        self.conv = nn.Conv2d(self.channels, self.out_channels, 3, stride=2, padding=1)
+
+    def forward(self, hidden_states):
+        assert hidden_states.shape[1] == self.channels
+        hidden_states = self.conv(hidden_states)
+
+        return hidden_states
+
+
+class CrossAttention(nn.Module):
+    def __init__(
+        self,
+        query_dim: int,
+        cross_attention_dim: Optional[int] = None,
+        heads: int = 8,
+        dim_head: int = 64,
+        upcast_attention: bool = False,
+    ):
+        super().__init__()
+        inner_dim = dim_head * heads
+        cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dim
+        self.upcast_attention = upcast_attention
+
+        self.scale = dim_head**-0.5
+        self.heads = heads
+
+        self.to_q = nn.Linear(query_dim, inner_dim, bias=False)
+        self.to_k = nn.Linear(cross_attention_dim, inner_dim, bias=False)
+        self.to_v = nn.Linear(cross_attention_dim, inner_dim, bias=False)
+
+        self.to_out = nn.ModuleList([])
+        self.to_out.append(nn.Linear(inner_dim, query_dim))
+        # no dropout here
+
+        self.use_memory_efficient_attention_xformers = False
+        self.use_memory_efficient_attention_mem_eff = False
+        self.use_sdpa = False
+
+    def set_use_memory_efficient_attention(self, xformers, mem_eff):
+        self.use_memory_efficient_attention_xformers = xformers
+        self.use_memory_efficient_attention_mem_eff = mem_eff
+
+    def set_use_sdpa(self, sdpa):
+        self.use_sdpa = sdpa
+
+    def reshape_heads_to_batch_dim(self, tensor):
+        batch_size, seq_len, dim = tensor.shape
+        head_size = self.heads
+        tensor = tensor.reshape(batch_size, seq_len, head_size, dim // head_size)
+        tensor = tensor.permute(0, 2, 1, 3).reshape(batch_size * head_size, seq_len, dim // head_size)
+        return tensor
+
+    def reshape_batch_dim_to_heads(self, tensor):
+        batch_size, seq_len, dim = tensor.shape
+        head_size = self.heads
+        tensor = tensor.reshape(batch_size // head_size, head_size, seq_len, dim)
+        tensor = tensor.permute(0, 2, 1, 3).reshape(batch_size // head_size, seq_len, dim * head_size)
+        return tensor
+
+    def forward(self, hidden_states, context=None, mask=None):
+        if self.use_memory_efficient_attention_xformers:
+            return self.forward_memory_efficient_xformers(hidden_states, context, mask)
+        if self.use_memory_efficient_attention_mem_eff:
+            return self.forward_memory_efficient_mem_eff(hidden_states, context, mask)
+        if self.use_sdpa:
+            return self.forward_sdpa(hidden_states, context, mask)
+
+        query = self.to_q(hidden_states)
+        context = context if context is not None else hidden_states
+        key = self.to_k(context)
+        value = self.to_v(context)
+
+        query = self.reshape_heads_to_batch_dim(query)
+        key = self.reshape_heads_to_batch_dim(key)
+        value = self.reshape_heads_to_batch_dim(value)
+
+        hidden_states = self._attention(query, key, value)
+
+        # linear proj
+        hidden_states = self.to_out[0](hidden_states)
+        # hidden_states = self.to_out[1](hidden_states)     # no dropout
+        return hidden_states
+
+    def _attention(self, query, key, value):
+        if self.upcast_attention:
+            query = query.float()
+            key = key.float()
+
+        attention_scores = torch.baddbmm(
+            torch.empty(query.shape[0], query.shape[1], key.shape[1], dtype=query.dtype, device=query.device),
+            query,
+            key.transpose(-1, -2),
+            beta=0,
+            alpha=self.scale,
+        )
+        attention_probs = attention_scores.softmax(dim=-1)
+
+        # cast back to the original dtype
+        attention_probs = attention_probs.to(value.dtype)
+
+        # compute attention output
+        hidden_states = torch.bmm(attention_probs, value)
+
+        # reshape hidden_states
+        hidden_states = self.reshape_batch_dim_to_heads(hidden_states)
+        return hidden_states
+
+    # TODO support Hypernetworks
+    def forward_memory_efficient_xformers(self, x, context=None, mask=None):
+        import xformers.ops
+
+        h = self.heads
+        q_in = self.to_q(x)
+        context = context if context is not None else x
+        context = context.to(x.dtype)
+        k_in = self.to_k(context)
+        v_in = self.to_v(context)
+
+        q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b n h d", h=h), (q_in, k_in, v_in))
+        del q_in, k_in, v_in
+
+        q = q.contiguous()
+        k = k.contiguous()
+        v = v.contiguous()
+        out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None)  # 最適なのを選んでくれる
+
+        out = rearrange(out, "b n h d -> b n (h d)", h=h)
+
+        out = self.to_out[0](out)
+        return out
+
+    def forward_memory_efficient_mem_eff(self, x, context=None, mask=None):
+        flash_func = FlashAttentionFunction
+
+        q_bucket_size = 512
+        k_bucket_size = 1024
+
+        h = self.heads
+        q = self.to_q(x)
+        context = context if context is not None else x
+        context = context.to(x.dtype)
+        k = self.to_k(context)
+        v = self.to_v(context)
+        del context, x
+
+        q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=h), (q, k, v))
+
+        out = flash_func.apply(q, k, v, mask, False, q_bucket_size, k_bucket_size)
+
+        out = rearrange(out, "b h n d -> b n (h d)")
+
+        out = self.to_out[0](out)
+        return out
+
+    def forward_sdpa(self, x, context=None, mask=None):
+        h = self.heads
+        q_in = self.to_q(x)
+        context = context if context is not None else x
+        context = context.to(x.dtype)
+        k_in = self.to_k(context)
+        v_in = self.to_v(context)
+
+        q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=h), (q_in, k_in, v_in))
+        del q_in, k_in, v_in
+
+        out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False)
+
+        out = rearrange(out, "b h n d -> b n (h d)", h=h)
+
+        out = self.to_out[0](out)
+        return out
+
+
+# feedforward
+class GEGLU(nn.Module):
+    r"""
+    A variant of the gated linear unit activation function from https://arxiv.org/abs/2002.05202.
+
+    Parameters:
+        dim_in (`int`): The number of channels in the input.
+        dim_out (`int`): The number of channels in the output.
+    """
+
+    def __init__(self, dim_in: int, dim_out: int):
+        super().__init__()
+        self.proj = nn.Linear(dim_in, dim_out * 2)
+
+    def gelu(self, gate):
+        if gate.device.type != "mps":
+            return F.gelu(gate)
+        # mps: gelu is not implemented for float16
+        return F.gelu(gate.to(dtype=torch.float32)).to(dtype=gate.dtype)
+
+    def forward(self, hidden_states):
+        hidden_states, gate = self.proj(hidden_states).chunk(2, dim=-1)
+        return hidden_states * self.gelu(gate)
+
+
+class FeedForward(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+    ):
+        super().__init__()
+        inner_dim = int(dim * 4)  # mult is always 4
+
+        self.net = nn.ModuleList([])
+        # project in
+        self.net.append(GEGLU(dim, inner_dim))
+        # project dropout
+        self.net.append(nn.Identity())  # nn.Dropout(0)) # dummy for dropout with 0
+        # project out
+        self.net.append(nn.Linear(inner_dim, dim))
+
+    def forward(self, hidden_states):
+        for module in self.net:
+            hidden_states = module(hidden_states)
+        return hidden_states
+
+
+class BasicTransformerBlock(nn.Module):
+    def __init__(
+        self, dim: int, num_attention_heads: int, attention_head_dim: int, cross_attention_dim: int, upcast_attention: bool = False
+    ):
+        super().__init__()
+
+        # 1. Self-Attn
+        self.attn1 = CrossAttention(
+            query_dim=dim,
+            cross_attention_dim=None,
+            heads=num_attention_heads,
+            dim_head=attention_head_dim,
+            upcast_attention=upcast_attention,
+        )
+        self.ff = FeedForward(dim)
+
+        # 2. Cross-Attn
+        self.attn2 = CrossAttention(
+            query_dim=dim,
+            cross_attention_dim=cross_attention_dim,
+            heads=num_attention_heads,
+            dim_head=attention_head_dim,
+            upcast_attention=upcast_attention,
+        )
+
+        self.norm1 = nn.LayerNorm(dim)
+        self.norm2 = nn.LayerNorm(dim)
+
+        # 3. Feed-forward
+        self.norm3 = nn.LayerNorm(dim)
+
+    def set_use_memory_efficient_attention(self, xformers: bool, mem_eff: bool):
+        self.attn1.set_use_memory_efficient_attention(xformers, mem_eff)
+        self.attn2.set_use_memory_efficient_attention(xformers, mem_eff)
+
+    def set_use_sdpa(self, sdpa: bool):
+        self.attn1.set_use_sdpa(sdpa)
+        self.attn2.set_use_sdpa(sdpa)
+
+    def forward(self, hidden_states, context=None, timestep=None):
+        # 1. Self-Attention
+        norm_hidden_states = self.norm1(hidden_states)
+
+        hidden_states = self.attn1(norm_hidden_states) + hidden_states
+
+        # 2. Cross-Attention
+        norm_hidden_states = self.norm2(hidden_states)
+        hidden_states = self.attn2(norm_hidden_states, context=context) + hidden_states
+
+        # 3. Feed-forward
+        hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
+
+        return hidden_states
+
+
+class Transformer2DModel(nn.Module):
+    def __init__(
+        self,
+        num_attention_heads: int = 16,
+        attention_head_dim: int = 88,
+        in_channels: Optional[int] = None,
+        cross_attention_dim: Optional[int] = None,
+        use_linear_projection: bool = False,
+        upcast_attention: bool = False,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.num_attention_heads = num_attention_heads
+        self.attention_head_dim = attention_head_dim
+        inner_dim = num_attention_heads * attention_head_dim
+        self.use_linear_projection = use_linear_projection
+
+        self.norm = torch.nn.GroupNorm(num_groups=TRANSFORMER_NORM_NUM_GROUPS, num_channels=in_channels, eps=1e-6, affine=True)
+
+        if use_linear_projection:
+            self.proj_in = nn.Linear(in_channels, inner_dim)
+        else:
+            self.proj_in = nn.Conv2d(in_channels, inner_dim, kernel_size=1, stride=1, padding=0)
+
+        self.transformer_blocks = nn.ModuleList(
+            [
+                BasicTransformerBlock(
+                    inner_dim,
+                    num_attention_heads,
+                    attention_head_dim,
+                    cross_attention_dim=cross_attention_dim,
+                    upcast_attention=upcast_attention,
+                )
+            ]
+        )
+
+        if use_linear_projection:
+            self.proj_out = nn.Linear(in_channels, inner_dim)
+        else:
+            self.proj_out = nn.Conv2d(inner_dim, in_channels, kernel_size=1, stride=1, padding=0)
+
+    def set_use_memory_efficient_attention(self, xformers, mem_eff):
+        for transformer in self.transformer_blocks:
+            transformer.set_use_memory_efficient_attention(xformers, mem_eff)
+
+    def set_use_sdpa(self, sdpa):
+        for transformer in self.transformer_blocks:
+            transformer.set_use_sdpa(sdpa)
+
+    def forward(self, hidden_states, encoder_hidden_states=None, timestep=None, return_dict: bool = True):
+        # 1. Input
+        batch, _, height, weight = hidden_states.shape
+        residual = hidden_states
+
+        hidden_states = self.norm(hidden_states)
+        if not self.use_linear_projection:
+            hidden_states = self.proj_in(hidden_states)
+            inner_dim = hidden_states.shape[1]
+            hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * weight, inner_dim)
+        else:
+            inner_dim = hidden_states.shape[1]
+            hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * weight, inner_dim)
+            hidden_states = self.proj_in(hidden_states)
+
+        # 2. Blocks
+        for block in self.transformer_blocks:
+            hidden_states = block(hidden_states, context=encoder_hidden_states, timestep=timestep)
+
+        # 3. Output
+        if not self.use_linear_projection:
+            hidden_states = hidden_states.reshape(batch, height, weight, inner_dim).permute(0, 3, 1, 2).contiguous()
+            hidden_states = self.proj_out(hidden_states)
+        else:
+            hidden_states = self.proj_out(hidden_states)
+            hidden_states = hidden_states.reshape(batch, height, weight, inner_dim).permute(0, 3, 1, 2).contiguous()
+
+        output = hidden_states + residual
+
+        if not return_dict:
+            return (output,)
+
+        return SampleOutput(sample=output)
+
+
+class CrossAttnDownBlock2D(nn.Module):
+    def __init__(
+        self,
+        in_channels: int,
+        out_channels: int,
+        add_downsample=True,
+        cross_attention_dim=1280,
+        attn_num_head_channels=1,
+        use_linear_projection=False,
+        upcast_attention=False,
+    ):
+        super().__init__()
+        self.has_cross_attention = True
+        resnets = []
+        attentions = []
+
+        self.attn_num_head_channels = attn_num_head_channels
+
+        for i in range(LAYERS_PER_BLOCK):
+            in_channels = in_channels if i == 0 else out_channels
+
+            resnets.append(ResnetBlock2D(in_channels=in_channels, out_channels=out_channels))
+            attentions.append(
+                Transformer2DModel(
+                    attn_num_head_channels,
+                    out_channels // attn_num_head_channels,
+                    in_channels=out_channels,
+                    cross_attention_dim=cross_attention_dim,
+                    use_linear_projection=use_linear_projection,
+                    upcast_attention=upcast_attention,
+                )
+            )
+        self.attentions = nn.ModuleList(attentions)
+        self.resnets = nn.ModuleList(resnets)
+
+        if add_downsample:
+            self.downsamplers = nn.ModuleList([Downsample2D(out_channels, out_channels)])
+        else:
+            self.downsamplers = None
+
+        self.gradient_checkpointing = False
+
+    def set_use_memory_efficient_attention(self, xformers, mem_eff):
+        for attn in self.attentions:
+            attn.set_use_memory_efficient_attention(xformers, mem_eff)
+
+    def set_use_sdpa(self, sdpa):
+        for attn in self.attentions:
+            attn.set_use_sdpa(sdpa)
+
+    def forward(self, hidden_states, temb=None, encoder_hidden_states=None):
+        output_states = ()
+
+        for resnet, attn in zip(self.resnets, self.attentions):
+            if self.training and self.gradient_checkpointing:
+
+                def create_custom_forward(module, return_dict=None):
+                    def custom_forward(*inputs):
+                        if return_dict is not None:
+                            return module(*inputs, return_dict=return_dict)
+                        else:
+                            return module(*inputs)
+
+                    return custom_forward
+
+                hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
+                hidden_states = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(attn, return_dict=False), hidden_states, encoder_hidden_states
+                )[0]
+            else:
+                hidden_states = resnet(hidden_states, temb)
+                hidden_states = attn(hidden_states, encoder_hidden_states=encoder_hidden_states).sample
+
+            output_states += (hidden_states,)
+
+        if self.downsamplers is not None:
+            for downsampler in self.downsamplers:
+                hidden_states = downsampler(hidden_states)
+
+            output_states += (hidden_states,)
+
+        return hidden_states, output_states
+
+
+class UNetMidBlock2DCrossAttn(nn.Module):
+    def __init__(
+        self,
+        in_channels: int,
+        attn_num_head_channels=1,
+        cross_attention_dim=1280,
+        use_linear_projection=False,
+    ):
+        super().__init__()
+
+        self.has_cross_attention = True
+        self.attn_num_head_channels = attn_num_head_channels
+
+        # Middle block has two resnets and one attention
+        resnets = [
+            ResnetBlock2D(
+                in_channels=in_channels,
+                out_channels=in_channels,
+            ),
+            ResnetBlock2D(
+                in_channels=in_channels,
+                out_channels=in_channels,
+            ),
+        ]
+        attentions = [
+            Transformer2DModel(
+                attn_num_head_channels,
+                in_channels // attn_num_head_channels,
+                in_channels=in_channels,
+                cross_attention_dim=cross_attention_dim,
+                use_linear_projection=use_linear_projection,
+            )
+        ]
+
+        self.attentions = nn.ModuleList(attentions)
+        self.resnets = nn.ModuleList(resnets)
+
+        self.gradient_checkpointing = False
+
+    def set_use_memory_efficient_attention(self, xformers, mem_eff):
+        for attn in self.attentions:
+            attn.set_use_memory_efficient_attention(xformers, mem_eff)
+
+    def set_use_sdpa(self, sdpa):
+        for attn in self.attentions:
+            attn.set_use_sdpa(sdpa)
+
+    def forward(self, hidden_states, temb=None, encoder_hidden_states=None):
+        for i, resnet in enumerate(self.resnets):
+            attn = None if i == 0 else self.attentions[i - 1]
+
+            if self.training and self.gradient_checkpointing:
+
+                def create_custom_forward(module, return_dict=None):
+                    def custom_forward(*inputs):
+                        if return_dict is not None:
+                            return module(*inputs, return_dict=return_dict)
+                        else:
+                            return module(*inputs)
+
+                    return custom_forward
+
+                if attn is not None:
+                    hidden_states = torch.utils.checkpoint.checkpoint(
+                        create_custom_forward(attn, return_dict=False), hidden_states, encoder_hidden_states
+                    )[0]
+
+                hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
+            else:
+                if attn is not None:
+                    hidden_states = attn(hidden_states, encoder_hidden_states).sample
+                hidden_states = resnet(hidden_states, temb)
+
+        return hidden_states
+
+
+class Upsample2D(nn.Module):
+    def __init__(self, channels, out_channels):
+        super().__init__()
+        self.channels = channels
+        self.out_channels = out_channels
+        self.conv = nn.Conv2d(self.channels, self.out_channels, 3, padding=1)
+
+    def forward(self, hidden_states, output_size):
+        assert hidden_states.shape[1] == self.channels
+
+        # Cast to float32 to as 'upsample_nearest2d_out_frame' op does not support bfloat16
+        # TODO(Suraj): Remove this cast once the issue is fixed in PyTorch
+        # https://github.com/pytorch/pytorch/issues/86679
+        dtype = hidden_states.dtype
+        if dtype == torch.bfloat16:
+            hidden_states = hidden_states.to(torch.float32)
+
+        # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/984
+        if hidden_states.shape[0] >= 64:
+            hidden_states = hidden_states.contiguous()
+
+        # if `output_size` is passed we force the interpolation output size and do not make use of `scale_factor=2`
+        if output_size is None:
+            hidden_states = F.interpolate(hidden_states, scale_factor=2.0, mode="nearest")
+        else:
+            hidden_states = F.interpolate(hidden_states, size=output_size, mode="nearest")
+
+        # If the input is bfloat16, we cast back to bfloat16
+        if dtype == torch.bfloat16:
+            hidden_states = hidden_states.to(dtype)
+
+        hidden_states = self.conv(hidden_states)
+
+        return hidden_states
+
+
+class UpBlock2D(nn.Module):
+    def __init__(
+        self,
+        in_channels: int,
+        prev_output_channel: int,
+        out_channels: int,
+        add_upsample=True,
+    ):
+        super().__init__()
+
+        self.has_cross_attention = False
+        resnets = []
+
+        for i in range(LAYERS_PER_BLOCK_UP):
+            res_skip_channels = in_channels if (i == LAYERS_PER_BLOCK_UP - 1) else out_channels
+            resnet_in_channels = prev_output_channel if i == 0 else out_channels
+
+            resnets.append(
+                ResnetBlock2D(
+                    in_channels=resnet_in_channels + res_skip_channels,
+                    out_channels=out_channels,
+                )
+            )
+
+        self.resnets = nn.ModuleList(resnets)
+
+        if add_upsample:
+            self.upsamplers = nn.ModuleList([Upsample2D(out_channels, out_channels)])
+        else:
+            self.upsamplers = None
+
+        self.gradient_checkpointing = False
+
+    def set_use_memory_efficient_attention(self, xformers, mem_eff):
+        pass
+
+    def set_use_sdpa(self, sdpa):
+        pass
+
+    def forward(self, hidden_states, res_hidden_states_tuple, temb=None, upsample_size=None):
+        for resnet in self.resnets:
+            # pop res hidden states
+            res_hidden_states = res_hidden_states_tuple[-1]
+            res_hidden_states_tuple = res_hidden_states_tuple[:-1]
+            hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
+
+            if self.training and self.gradient_checkpointing:
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        return module(*inputs)
+
+                    return custom_forward
+
+                hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
+            else:
+                hidden_states = resnet(hidden_states, temb)
+
+        if self.upsamplers is not None:
+            for upsampler in self.upsamplers:
+                hidden_states = upsampler(hidden_states, upsample_size)
+
+        return hidden_states
+
+
+class CrossAttnUpBlock2D(nn.Module):
+    def __init__(
+        self,
+        in_channels: int,
+        out_channels: int,
+        prev_output_channel: int,
+        attn_num_head_channels=1,
+        cross_attention_dim=1280,
+        add_upsample=True,
+        use_linear_projection=False,
+        upcast_attention=False,
+    ):
+        super().__init__()
+        resnets = []
+        attentions = []
+
+        self.has_cross_attention = True
+        self.attn_num_head_channels = attn_num_head_channels
+
+        for i in range(LAYERS_PER_BLOCK_UP):
+            res_skip_channels = in_channels if (i == LAYERS_PER_BLOCK_UP - 1) else out_channels
+            resnet_in_channels = prev_output_channel if i == 0 else out_channels
+
+            resnets.append(
+                ResnetBlock2D(
+                    in_channels=resnet_in_channels + res_skip_channels,
+                    out_channels=out_channels,
+                )
+            )
+            attentions.append(
+                Transformer2DModel(
+                    attn_num_head_channels,
+                    out_channels // attn_num_head_channels,
+                    in_channels=out_channels,
+                    cross_attention_dim=cross_attention_dim,
+                    use_linear_projection=use_linear_projection,
+                    upcast_attention=upcast_attention,
+                )
+            )
+
+        self.attentions = nn.ModuleList(attentions)
+        self.resnets = nn.ModuleList(resnets)
+
+        if add_upsample:
+            self.upsamplers = nn.ModuleList([Upsample2D(out_channels, out_channels)])
+        else:
+            self.upsamplers = None
+
+        self.gradient_checkpointing = False
+
+    def set_use_memory_efficient_attention(self, xformers, mem_eff):
+        for attn in self.attentions:
+            attn.set_use_memory_efficient_attention(xformers, mem_eff)
+
+    def set_use_sdpa(self, spda):
+        for attn in self.attentions:
+            attn.set_use_sdpa(spda)
+
+    def forward(
+        self,
+        hidden_states,
+        res_hidden_states_tuple,
+        temb=None,
+        encoder_hidden_states=None,
+        upsample_size=None,
+    ):
+        for resnet, attn in zip(self.resnets, self.attentions):
+            # pop res hidden states
+            res_hidden_states = res_hidden_states_tuple[-1]
+            res_hidden_states_tuple = res_hidden_states_tuple[:-1]
+            hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
+
+            if self.training and self.gradient_checkpointing:
+
+                def create_custom_forward(module, return_dict=None):
+                    def custom_forward(*inputs):
+                        if return_dict is not None:
+                            return module(*inputs, return_dict=return_dict)
+                        else:
+                            return module(*inputs)
+
+                    return custom_forward
+
+                hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb)
+                hidden_states = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(attn, return_dict=False), hidden_states, encoder_hidden_states
+                )[0]
+            else:
+                hidden_states = resnet(hidden_states, temb)
+                hidden_states = attn(hidden_states, encoder_hidden_states=encoder_hidden_states).sample
+
+        if self.upsamplers is not None:
+            for upsampler in self.upsamplers:
+                hidden_states = upsampler(hidden_states, upsample_size)
+
+        return hidden_states
+
+
+def get_down_block(
+    down_block_type,
+    in_channels,
+    out_channels,
+    add_downsample,
+    attn_num_head_channels,
+    cross_attention_dim,
+    use_linear_projection,
+    upcast_attention,
+):
+    if down_block_type == "DownBlock2D":
+        return DownBlock2D(
+            in_channels=in_channels,
+            out_channels=out_channels,
+            add_downsample=add_downsample,
+        )
+    elif down_block_type == "CrossAttnDownBlock2D":
+        return CrossAttnDownBlock2D(
+            in_channels=in_channels,
+            out_channels=out_channels,
+            add_downsample=add_downsample,
+            cross_attention_dim=cross_attention_dim,
+            attn_num_head_channels=attn_num_head_channels,
+            use_linear_projection=use_linear_projection,
+            upcast_attention=upcast_attention,
+        )
+
+
+def get_up_block(
+    up_block_type,
+    in_channels,
+    out_channels,
+    prev_output_channel,
+    add_upsample,
+    attn_num_head_channels,
+    cross_attention_dim=None,
+    use_linear_projection=False,
+    upcast_attention=False,
+):
+    if up_block_type == "UpBlock2D":
+        return UpBlock2D(
+            in_channels=in_channels,
+            prev_output_channel=prev_output_channel,
+            out_channels=out_channels,
+            add_upsample=add_upsample,
+        )
+    elif up_block_type == "CrossAttnUpBlock2D":
+        return CrossAttnUpBlock2D(
+            in_channels=in_channels,
+            out_channels=out_channels,
+            prev_output_channel=prev_output_channel,
+            attn_num_head_channels=attn_num_head_channels,
+            cross_attention_dim=cross_attention_dim,
+            add_upsample=add_upsample,
+            use_linear_projection=use_linear_projection,
+            upcast_attention=upcast_attention,
+        )
+
+
+class UNet2DConditionModel(nn.Module):
+    _supports_gradient_checkpointing = True
+
+    def __init__(
+        self,
+        sample_size: Optional[int] = None,
+        attention_head_dim: Union[int, Tuple[int]] = 8,
+        cross_attention_dim: int = 1280,
+        use_linear_projection: bool = False,
+        upcast_attention: bool = False,
+        **kwargs,
+    ):
+        super().__init__()
+        assert sample_size is not None, "sample_size must be specified"
+        print(
+            f"UNet2DConditionModel: {sample_size}, {attention_head_dim}, {cross_attention_dim}, {use_linear_projection}, {upcast_attention}"
+        )
+
+        # 外部からの参照用に定義しておく
+        self.in_channels = IN_CHANNELS
+        self.out_channels = OUT_CHANNELS
+
+        self.sample_size = sample_size
+        self.prepare_config()
+
+        # state_dictの書式が変わるのでmoduleの持ち方は変えられない
+
+        # input
+        self.conv_in = nn.Conv2d(IN_CHANNELS, BLOCK_OUT_CHANNELS[0], kernel_size=3, padding=(1, 1))
+
+        # time
+        self.time_proj = Timesteps(BLOCK_OUT_CHANNELS[0], TIME_EMBED_FLIP_SIN_TO_COS, TIME_EMBED_FREQ_SHIFT)
+
+        self.time_embedding = TimestepEmbedding(TIMESTEP_INPUT_DIM, TIME_EMBED_DIM)
+
+        self.down_blocks = nn.ModuleList([])
+        self.mid_block = None
+        self.up_blocks = nn.ModuleList([])
+
+        if isinstance(attention_head_dim, int):
+            attention_head_dim = (attention_head_dim,) * 4
+
+        # down
+        output_channel = BLOCK_OUT_CHANNELS[0]
+        for i, down_block_type in enumerate(DOWN_BLOCK_TYPES):
+            input_channel = output_channel
+            output_channel = BLOCK_OUT_CHANNELS[i]
+            is_final_block = i == len(BLOCK_OUT_CHANNELS) - 1
+
+            down_block = get_down_block(
+                down_block_type,
+                in_channels=input_channel,
+                out_channels=output_channel,
+                add_downsample=not is_final_block,
+                attn_num_head_channels=attention_head_dim[i],
+                cross_attention_dim=cross_attention_dim,
+                use_linear_projection=use_linear_projection,
+                upcast_attention=upcast_attention,
+            )
+            self.down_blocks.append(down_block)
+
+        # mid
+        self.mid_block = UNetMidBlock2DCrossAttn(
+            in_channels=BLOCK_OUT_CHANNELS[-1],
+            attn_num_head_channels=attention_head_dim[-1],
+            cross_attention_dim=cross_attention_dim,
+            use_linear_projection=use_linear_projection,
+        )
+
+        # count how many layers upsample the images
+        self.num_upsamplers = 0
+
+        # up
+        reversed_block_out_channels = list(reversed(BLOCK_OUT_CHANNELS))
+        reversed_attention_head_dim = list(reversed(attention_head_dim))
+        output_channel = reversed_block_out_channels[0]
+        for i, up_block_type in enumerate(UP_BLOCK_TYPES):
+            is_final_block = i == len(BLOCK_OUT_CHANNELS) - 1
+
+            prev_output_channel = output_channel
+            output_channel = reversed_block_out_channels[i]
+            input_channel = reversed_block_out_channels[min(i + 1, len(BLOCK_OUT_CHANNELS) - 1)]
+
+            # add upsample block for all BUT final layer
+            if not is_final_block:
+                add_upsample = True
+                self.num_upsamplers += 1
+            else:
+                add_upsample = False
+
+            up_block = get_up_block(
+                up_block_type,
+                in_channels=input_channel,
+                out_channels=output_channel,
+                prev_output_channel=prev_output_channel,
+                add_upsample=add_upsample,
+                attn_num_head_channels=reversed_attention_head_dim[i],
+                cross_attention_dim=cross_attention_dim,
+                use_linear_projection=use_linear_projection,
+                upcast_attention=upcast_attention,
+            )
+            self.up_blocks.append(up_block)
+            prev_output_channel = output_channel
+
+        # out
+        self.conv_norm_out = nn.GroupNorm(num_channels=BLOCK_OUT_CHANNELS[0], num_groups=NORM_GROUPS, eps=NORM_EPS)
+        self.conv_act = nn.SiLU()
+        self.conv_out = nn.Conv2d(BLOCK_OUT_CHANNELS[0], OUT_CHANNELS, kernel_size=3, padding=1)
+
+    # region diffusers compatibility
+    def prepare_config(self):
+        self.config = SimpleNamespace()
+
+    @property
+    def dtype(self) -> torch.dtype:
+        # `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype).
+        return get_parameter_dtype(self)
+
+    @property
+    def device(self) -> torch.device:
+        # `torch.device`: The device on which the module is (assuming that all the module parameters are on the same device).
+        return get_parameter_device(self)
+
+    def set_attention_slice(self, slice_size):
+        raise NotImplementedError("Attention slicing is not supported for this model.")
+
+    def is_gradient_checkpointing(self) -> bool:
+        return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules())
+
+    def enable_gradient_checkpointing(self):
+        self.set_gradient_checkpointing(value=True)
+
+    def disable_gradient_checkpointing(self):
+        self.set_gradient_checkpointing(value=False)
+
+    def set_use_memory_efficient_attention(self, xformers: bool, mem_eff: bool) -> None:
+        modules = self.down_blocks + [self.mid_block] + self.up_blocks
+        for module in modules:
+            module.set_use_memory_efficient_attention(xformers, mem_eff)
+
+    def set_use_sdpa(self, sdpa: bool) -> None:
+        modules = self.down_blocks + [self.mid_block] + self.up_blocks
+        for module in modules:
+            module.set_use_sdpa(sdpa)
+
+    def set_gradient_checkpointing(self, value=False):
+        modules = self.down_blocks + [self.mid_block] + self.up_blocks
+        for module in modules:
+            print(module.__class__.__name__, module.gradient_checkpointing, "->", value)
+            module.gradient_checkpointing = value
+
+    # endregion
+
+    def forward(
+        self,
+        sample: torch.FloatTensor,
+        timestep: Union[torch.Tensor, float, int],
+        encoder_hidden_states: torch.Tensor,
+        class_labels: Optional[torch.Tensor] = None,
+        return_dict: bool = True,
+        down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
+        mid_block_additional_residual: Optional[torch.Tensor] = None,
+    ) -> Union[Dict, Tuple]:
+        r"""
+        Args:
+            sample (`torch.FloatTensor`): (batch, channel, height, width) noisy inputs tensor
+            timestep (`torch.FloatTensor` or `float` or `int`): (batch) timesteps
+            encoder_hidden_states (`torch.FloatTensor`): (batch, sequence_length, feature_dim) encoder hidden states
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a dict instead of a plain tuple.
+
+        Returns:
+            `SampleOutput` or `tuple`:
+            `SampleOutput` if `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is the sample tensor.
+        """
+        # By default samples have to be AT least a multiple of the overall upsampling factor.
+        # The overall upsampling factor is equal to 2 ** (# num of upsampling layears).
+        # However, the upsampling interpolation output size can be forced to fit any upsampling size
+        # on the fly if necessary.
+        # デフォルトではサンプルは「2^アップサンプルの数」、つまり64の倍数である必要がある
+        # ただそれ以外のサイズにも対応できるように、必要ならアップサンプルのサイズを変更する
+        # 多分画質が悪くなるので、64で割り切れるようにしておくのが良い
+        default_overall_up_factor = 2**self.num_upsamplers
+
+        # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
+        # 64で割り切れないときはupsamplerにサイズを伝える
+        forward_upsample_size = False
+        upsample_size = None
+
+        if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):
+            # logger.info("Forward upsample size to force interpolation output size.")
+            forward_upsample_size = True
+
+        # 1. time
+        timesteps = timestep
+        timesteps = self.handle_unusual_timesteps(sample, timesteps)  # 変な時だけ処理
+
+        t_emb = self.time_proj(timesteps)
+
+        # timesteps does not contain any weights and will always return f32 tensors
+        # but time_embedding might actually be running in fp16. so we need to cast here.
+        # there might be better ways to encapsulate this.
+        # timestepsは重みを含まないので常にfloat32のテンソルを返す
+        # しかしtime_embeddingはfp16で動いているかもしれないので、ここでキャストする必要がある
+        # time_projでキャストしておけばいいんじゃね？
+        t_emb = t_emb.to(dtype=self.dtype)
+        emb = self.time_embedding(t_emb)
+
+        # 2. pre-process
+        sample = self.conv_in(sample)
+
+        # 3. down
+        down_block_res_samples = (sample,)
+        for downsample_block in self.down_blocks:
+            # downblockはforwardで必ずencoder_hidden_statesを受け取るようにしても良さそうだけど、
+            # まあこちらのほうがわかりやすいかもしれない
+            if downsample_block.has_cross_attention:
+                sample, res_samples = downsample_block(
+                    hidden_states=sample,
+                    temb=emb,
+                    encoder_hidden_states=encoder_hidden_states,
+                )
+            else:
+                sample, res_samples = downsample_block(hidden_states=sample, temb=emb)
+
+            down_block_res_samples += res_samples
+
+        # skip connectionにControlNetの出力を追加する
+        if down_block_additional_residuals is not None:
+            down_block_res_samples = list(down_block_res_samples)
+            for i in range(len(down_block_res_samples)):
+                down_block_res_samples[i] += down_block_additional_residuals[i]
+            down_block_res_samples = tuple(down_block_res_samples)
+
+        # 4. mid
+        sample = self.mid_block(sample, emb, encoder_hidden_states=encoder_hidden_states)
+
+        # ControlNetの出力を追加する
+        if mid_block_additional_residual is not None:
+            sample += mid_block_additional_residual
+
+        # 5. up
+        for i, upsample_block in enumerate(self.up_blocks):
+            is_final_block = i == len(self.up_blocks) - 1
+
+            res_samples = down_block_res_samples[-len(upsample_block.resnets) :]
+            down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]  # skip connection
+
+            # if we have not reached the final block and need to forward the upsample size, we do it here
+            # 前述のように最後のブロック以外ではupsample_sizeを伝える
+            if not is_final_block and forward_upsample_size:
+                upsample_size = down_block_res_samples[-1].shape[2:]
+
+            if upsample_block.has_cross_attention:
+                sample = upsample_block(
+                    hidden_states=sample,
+                    temb=emb,
+                    res_hidden_states_tuple=res_samples,
+                    encoder_hidden_states=encoder_hidden_states,
+                    upsample_size=upsample_size,
+                )
+            else:
+                sample = upsample_block(
+                    hidden_states=sample, temb=emb, res_hidden_states_tuple=res_samples, upsample_size=upsample_size
+                )
+
+        # 6. post-process
+        sample = self.conv_norm_out(sample)
+        sample = self.conv_act(sample)
+        sample = self.conv_out(sample)
+
+        if not return_dict:
+            return (sample,)
+
+        return SampleOutput(sample=sample)
+
+    def handle_unusual_timesteps(self, sample, timesteps):
+        r"""
+        timestampsがTensorでない場合、Tensorに変換する。またOnnx/Core MLと互換性のあるようにbatchサイズまでbroadcastする。
+        """
+        if not torch.is_tensor(timesteps):
+            # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
+            # This would be a good case for the `match` statement (Python 3.10+)
+            is_mps = sample.device.type == "mps"
+            if isinstance(timesteps, float):
+                dtype = torch.float32 if is_mps else torch.float64
+            else:
+                dtype = torch.int32 if is_mps else torch.int64
+            timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
+        elif len(timesteps.shape) == 0:
+            timesteps = timesteps[None].to(sample.device)
+
+        # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+        timesteps = timesteps.expand(sample.shape[0])
+
+        return timesteps
diff --git a/library/resize_lora_gui.py b/library/resize_lora_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..e002b749399f2c5ec26d7c5f6832b73d314f423e
--- /dev/null
+++ b/library/resize_lora_gui.py
@@ -0,0 +1,184 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import get_saveasfilename_path, get_file_path
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+
+
+def resize_lora(
+    model,
+    new_rank,
+    save_to,
+    save_precision,
+    device,
+    dynamic_method,
+    dynamic_param,
+    verbose,
+):
+    # Check for caption_text_input
+    if model == '':
+        msgbox('Invalid model file')
+        return
+
+    # Check if source model exist
+    if not os.path.isfile(model):
+        msgbox('The provided model is not a file')
+        return
+
+    if dynamic_method == 'sv_ratio':
+        if float(dynamic_param) < 2:
+            msgbox(
+                f'Dynamic parameter for {dynamic_method} need to be 2 or greater...'
+            )
+            return
+
+    if dynamic_method == 'sv_fro' or dynamic_method == 'sv_cumulative':
+        if float(dynamic_param) < 0 or float(dynamic_param) > 1:
+            msgbox(
+                f'Dynamic parameter for {dynamic_method} need to be between 0 and 1...'
+            )
+            return
+
+    # Check if save_to end with one of the defines extension. If not add .safetensors.
+    if not save_to.endswith(('.pt', '.safetensors')):
+        save_to += '.safetensors'
+
+    if device == '':
+        device = 'cuda'
+
+    run_cmd = f'{PYTHON} "{os.path.join("networks","resize_lora.py")}"'
+    run_cmd += f' --save_precision {save_precision}'
+    run_cmd += f' --save_to "{save_to}"'
+    run_cmd += f' --model "{model}"'
+    run_cmd += f' --new_rank {new_rank}'
+    run_cmd += f' --device {device}'
+    if not dynamic_method == 'None':
+        run_cmd += f' --dynamic_method {dynamic_method}'
+        run_cmd += f' --dynamic_param {dynamic_param}'
+    if verbose:
+        run_cmd += f' --verbose'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+    log.info('Done resizing...')
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_resize_lora_tab(headless=False):
+    with gr.Tab('Resize LoRA'):
+        gr.Markdown('This utility can resize a LoRA.')
+
+        lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False)
+        lora_ext_name = gr.Textbox(value='LoRA model types', visible=False)
+
+        with gr.Row():
+            model = gr.Textbox(
+                label='Source LoRA',
+                placeholder='Path to the LoRA to resize',
+                interactive=True,
+            )
+            button_lora_a_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_a_model_file.click(
+                get_file_path,
+                inputs=[model, lora_ext, lora_ext_name],
+                outputs=model,
+                show_progress=False,
+            )
+        with gr.Row():
+            save_to = gr.Textbox(
+                label='Save to',
+                placeholder='path for the LoRA file to save...',
+                interactive=True,
+            )
+            button_save_to = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_save_to.click(
+                get_saveasfilename_path,
+                inputs=[save_to, lora_ext, lora_ext_name],
+                outputs=save_to,
+                show_progress=False,
+            )
+        with gr.Row():
+            new_rank = gr.Slider(
+                label='Desired LoRA rank',
+                minimum=1,
+                maximum=1024,
+                step=1,
+                value=4,
+                interactive=True,
+            )
+            dynamic_method = gr.Dropdown(
+                choices=['None', 'sv_ratio', 'sv_fro', 'sv_cumulative'],
+                value='sv_fro',
+                label='Dynamic method',
+                interactive=True,
+            )
+            dynamic_param = gr.Textbox(
+                label='Dynamic parameter',
+                value='0.9',
+                interactive=True,
+                placeholder='Value for the dynamic method selected.',
+            )
+        with gr.Row():
+            
+            verbose = gr.Checkbox(label='Verbose', value=True)
+            save_precision = gr.Dropdown(
+                label='Save precision',
+                choices=['fp16', 'bf16', 'float'],
+                value='fp16',
+                interactive=True,
+            )
+            device = gr.Dropdown(
+                label='Device',
+                choices=[
+                    'cpu',
+                    'cuda',
+                ],
+                value='cuda',
+                interactive=True,
+            )
+
+        convert_button = gr.Button('Resize model')
+
+        convert_button.click(
+            resize_lora,
+            inputs=[
+                model,
+                new_rank,
+                save_to,
+                save_precision,
+                device,
+                dynamic_method,
+                dynamic_param,
+                verbose,
+            ],
+            show_progress=False,
+        )
diff --git a/library/sdxl_lpw_stable_diffusion.py b/library/sdxl_lpw_stable_diffusion.py
new file mode 100644
index 0000000000000000000000000000000000000000..99b0bc8d51c6b20ff18dbe762617f1d60f1bf9d6
--- /dev/null
+++ b/library/sdxl_lpw_stable_diffusion.py
@@ -0,0 +1,1347 @@
+# copy from https://github.com/huggingface/diffusers/blob/main/examples/community/lpw_stable_diffusion.py
+# and modify to support SD2.x
+
+import inspect
+import re
+from typing import Callable, List, Optional, Union
+
+import numpy as np
+import PIL.Image
+import torch
+from packaging import version
+from tqdm import tqdm
+from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer
+
+from diffusers import SchedulerMixin, StableDiffusionPipeline
+from diffusers.models import AutoencoderKL, UNet2DConditionModel
+from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker
+from diffusers.utils import logging
+from PIL import Image
+
+from library import sdxl_model_util, sdxl_train_util
+
+
+try:
+    from diffusers.utils import PIL_INTERPOLATION
+except ImportError:
+    if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"):
+        PIL_INTERPOLATION = {
+            "linear": PIL.Image.Resampling.BILINEAR,
+            "bilinear": PIL.Image.Resampling.BILINEAR,
+            "bicubic": PIL.Image.Resampling.BICUBIC,
+            "lanczos": PIL.Image.Resampling.LANCZOS,
+            "nearest": PIL.Image.Resampling.NEAREST,
+        }
+    else:
+        PIL_INTERPOLATION = {
+            "linear": PIL.Image.LINEAR,
+            "bilinear": PIL.Image.BILINEAR,
+            "bicubic": PIL.Image.BICUBIC,
+            "lanczos": PIL.Image.LANCZOS,
+            "nearest": PIL.Image.NEAREST,
+        }
+# ------------------------------------------------------------------------------
+
+logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+
+re_attention = re.compile(
+    r"""
+\\\(|
+\\\)|
+\\\[|
+\\]|
+\\\\|
+\\|
+\(|
+\[|
+:([+-]?[.\d]+)\)|
+\)|
+]|
+[^\\()\[\]:]+|
+:
+""",
+    re.X,
+)
+
+
+def parse_prompt_attention(text):
+    """
+    Parses a string with attention tokens and returns a list of pairs: text and its associated weight.
+    Accepted tokens are:
+      (abc) - increases attention to abc by a multiplier of 1.1
+      (abc:3.12) - increases attention to abc by a multiplier of 3.12
+      [abc] - decreases attention to abc by a multiplier of 1.1
+      \( - literal character '('
+      \[ - literal character '['
+      \) - literal character ')'
+      \] - literal character ']'
+      \\ - literal character '\'
+      anything else - just text
+    >>> parse_prompt_attention('normal text')
+    [['normal text', 1.0]]
+    >>> parse_prompt_attention('an (important) word')
+    [['an ', 1.0], ['important', 1.1], [' word', 1.0]]
+    >>> parse_prompt_attention('(unbalanced')
+    [['unbalanced', 1.1]]
+    >>> parse_prompt_attention('\(literal\]')
+    [['(literal]', 1.0]]
+    >>> parse_prompt_attention('(unnecessary)(parens)')
+    [['unnecessaryparens', 1.1]]
+    >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')
+    [['a ', 1.0],
+     ['house', 1.5730000000000004],
+     [' ', 1.1],
+     ['on', 1.0],
+     [' a ', 1.1],
+     ['hill', 0.55],
+     [', sun, ', 1.1],
+     ['sky', 1.4641000000000006],
+     ['.', 1.1]]
+    """
+
+    res = []
+    round_brackets = []
+    square_brackets = []
+
+    round_bracket_multiplier = 1.1
+    square_bracket_multiplier = 1 / 1.1
+
+    def multiply_range(start_position, multiplier):
+        for p in range(start_position, len(res)):
+            res[p][1] *= multiplier
+
+    for m in re_attention.finditer(text):
+        text = m.group(0)
+        weight = m.group(1)
+
+        if text.startswith("\\"):
+            res.append([text[1:], 1.0])
+        elif text == "(":
+            round_brackets.append(len(res))
+        elif text == "[":
+            square_brackets.append(len(res))
+        elif weight is not None and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), float(weight))
+        elif text == ")" and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), round_bracket_multiplier)
+        elif text == "]" and len(square_brackets) > 0:
+            multiply_range(square_brackets.pop(), square_bracket_multiplier)
+        else:
+            res.append([text, 1.0])
+
+    for pos in round_brackets:
+        multiply_range(pos, round_bracket_multiplier)
+
+    for pos in square_brackets:
+        multiply_range(pos, square_bracket_multiplier)
+
+    if len(res) == 0:
+        res = [["", 1.0]]
+
+    # merge runs of identical weights
+    i = 0
+    while i + 1 < len(res):
+        if res[i][1] == res[i + 1][1]:
+            res[i][0] += res[i + 1][0]
+            res.pop(i + 1)
+        else:
+            i += 1
+
+    return res
+
+
+def get_prompts_with_weights(pipe: StableDiffusionPipeline, prompt: List[str], max_length: int):
+    r"""
+    Tokenize a list of prompts and return its tokens with weights of each token.
+
+    No padding, starting or ending token is included.
+    """
+    tokens = []
+    weights = []
+    truncated = False
+    for text in prompt:
+        texts_and_weights = parse_prompt_attention(text)
+        text_token = []
+        text_weight = []
+        for word, weight in texts_and_weights:
+            # tokenize and discard the starting and the ending token
+            token = pipe.tokenizer(word).input_ids[1:-1]
+            text_token += token
+            # copy the weight by length of token
+            text_weight += [weight] * len(token)
+            # stop if the text is too long (longer than truncation limit)
+            if len(text_token) > max_length:
+                truncated = True
+                break
+        # truncate
+        if len(text_token) > max_length:
+            truncated = True
+            text_token = text_token[:max_length]
+            text_weight = text_weight[:max_length]
+        tokens.append(text_token)
+        weights.append(text_weight)
+    if truncated:
+        logger.warning("Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples")
+    return tokens, weights
+
+
+def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, pad, no_boseos_middle=True, chunk_length=77):
+    r"""
+    Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length.
+    """
+    max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)
+    weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length
+    for i in range(len(tokens)):
+        tokens[i] = [bos] + tokens[i] + [eos] + [pad] * (max_length - 2 - len(tokens[i]))
+        if no_boseos_middle:
+            weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))
+        else:
+            w = []
+            if len(weights[i]) == 0:
+                w = [1.0] * weights_length
+            else:
+                for j in range(max_embeddings_multiples):
+                    w.append(1.0)  # weight for starting token in this chunk
+                    w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]
+                    w.append(1.0)  # weight for ending token in this chunk
+                w += [1.0] * (weights_length - len(w))
+            weights[i] = w[:]
+
+    return tokens, weights
+
+
+def get_hidden_states(text_encoder, input_ids, is_sdxl_text_encoder2: bool, device):
+    if not is_sdxl_text_encoder2:
+        # text_encoder1: same as SD1/2
+        enc_out = text_encoder(input_ids.to(text_encoder.device), output_hidden_states=True, return_dict=True)
+        hidden_states = enc_out["hidden_states"][11]
+        pool = None
+    else:
+        # text_encoder2
+        enc_out = text_encoder(input_ids.to(text_encoder.device), output_hidden_states=True, return_dict=True)
+        hidden_states = enc_out["hidden_states"][-2]  # penuultimate layer
+        pool = enc_out["text_embeds"]
+    hidden_states = hidden_states.to(device)
+    if pool is not None:
+        pool = pool.to(device)
+    return hidden_states, pool
+
+
+def get_unweighted_text_embeddings(
+    pipe: StableDiffusionPipeline,
+    text_input: torch.Tensor,
+    chunk_length: int,
+    clip_skip: int,
+    eos: int,
+    pad: int,
+    is_sdxl_text_encoder2: bool,
+    no_boseos_middle: Optional[bool] = True,
+):
+    """
+    When the length of tokens is a multiple of the capacity of the text encoder,
+    it should be split into chunks and sent to the text encoder individually.
+    """
+    max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)
+    text_pool = None
+    if max_embeddings_multiples > 1:
+        text_embeddings = []
+        for i in range(max_embeddings_multiples):
+            # extract the i-th chunk
+            text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()
+
+            # cover the head and the tail by the starting and the ending tokens
+            text_input_chunk[:, 0] = text_input[0, 0]
+            if pad == eos:  # v1
+                text_input_chunk[:, -1] = text_input[0, -1]
+            else:  # v2
+                for j in range(len(text_input_chunk)):
+                    if text_input_chunk[j, -1] != eos and text_input_chunk[j, -1] != pad:  # 最後に普通の文字がある
+                        text_input_chunk[j, -1] = eos
+                    if text_input_chunk[j, 1] == pad:  # BOSだけであとはPAD
+                        text_input_chunk[j, 1] = eos
+
+            text_embedding, current_text_pool = get_hidden_states(
+                pipe.text_encoder, text_input_chunk, is_sdxl_text_encoder2, pipe.device
+            )
+            if text_pool is None:
+                text_pool = current_text_pool
+
+            if no_boseos_middle:
+                if i == 0:
+                    # discard the ending token
+                    text_embedding = text_embedding[:, :-1]
+                elif i == max_embeddings_multiples - 1:
+                    # discard the starting token
+                    text_embedding = text_embedding[:, 1:]
+                else:
+                    # discard both starting and ending tokens
+                    text_embedding = text_embedding[:, 1:-1]
+
+            text_embeddings.append(text_embedding)
+        text_embeddings = torch.concat(text_embeddings, axis=1)
+    else:
+        text_embeddings, text_pool = get_hidden_states(pipe.text_encoder, text_input, is_sdxl_text_encoder2, pipe.device)
+    return text_embeddings, text_pool
+
+
+def get_weighted_text_embeddings(
+    pipe,  # : SdxlStableDiffusionLongPromptWeightingPipeline,
+    prompt: Union[str, List[str]],
+    uncond_prompt: Optional[Union[str, List[str]]] = None,
+    max_embeddings_multiples: Optional[int] = 3,
+    no_boseos_middle: Optional[bool] = False,
+    skip_parsing: Optional[bool] = False,
+    skip_weighting: Optional[bool] = False,
+    clip_skip=None,
+    is_sdxl_text_encoder2=False,
+):
+    r"""
+    Prompts can be assigned with local weights using brackets. For example,
+    prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful',
+    and the embedding tokens corresponding to the words get multiplied by a constant, 1.1.
+
+    Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean.
+
+    Args:
+        pipe (`StableDiffusionPipeline`):
+            Pipe to provide access to the tokenizer and the text encoder.
+        prompt (`str` or `List[str]`):
+            The prompt or prompts to guide the image generation.
+        uncond_prompt (`str` or `List[str]`):
+            The unconditional prompt or prompts for guide the image generation. If unconditional prompt
+            is provided, the embeddings of prompt and uncond_prompt are concatenated.
+        max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+            The max multiple length of prompt embeddings compared to the max output length of text encoder.
+        no_boseos_middle (`bool`, *optional*, defaults to `False`):
+            If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and
+            ending token in each of the chunk in the middle.
+        skip_parsing (`bool`, *optional*, defaults to `False`):
+            Skip the parsing of brackets.
+        skip_weighting (`bool`, *optional*, defaults to `False`):
+            Skip the weighting. When the parsing is skipped, it is forced True.
+    """
+    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+    if isinstance(prompt, str):
+        prompt = [prompt]
+
+    if not skip_parsing:
+        prompt_tokens, prompt_weights = get_prompts_with_weights(pipe, prompt, max_length - 2)
+        if uncond_prompt is not None:
+            if isinstance(uncond_prompt, str):
+                uncond_prompt = [uncond_prompt]
+            uncond_tokens, uncond_weights = get_prompts_with_weights(pipe, uncond_prompt, max_length - 2)
+    else:
+        prompt_tokens = [token[1:-1] for token in pipe.tokenizer(prompt, max_length=max_length, truncation=True).input_ids]
+        prompt_weights = [[1.0] * len(token) for token in prompt_tokens]
+        if uncond_prompt is not None:
+            if isinstance(uncond_prompt, str):
+                uncond_prompt = [uncond_prompt]
+            uncond_tokens = [
+                token[1:-1] for token in pipe.tokenizer(uncond_prompt, max_length=max_length, truncation=True).input_ids
+            ]
+            uncond_weights = [[1.0] * len(token) for token in uncond_tokens]
+
+    # round up the longest length of tokens to a multiple of (model_max_length - 2)
+    max_length = max([len(token) for token in prompt_tokens])
+    if uncond_prompt is not None:
+        max_length = max(max_length, max([len(token) for token in uncond_tokens]))
+
+    max_embeddings_multiples = min(
+        max_embeddings_multiples,
+        (max_length - 1) // (pipe.tokenizer.model_max_length - 2) + 1,
+    )
+    max_embeddings_multiples = max(1, max_embeddings_multiples)
+    max_length = (pipe.tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+
+    # pad the length of tokens and weights
+    bos = pipe.tokenizer.bos_token_id
+    eos = pipe.tokenizer.eos_token_id
+    pad = pipe.tokenizer.pad_token_id
+    prompt_tokens, prompt_weights = pad_tokens_and_weights(
+        prompt_tokens,
+        prompt_weights,
+        max_length,
+        bos,
+        eos,
+        pad,
+        no_boseos_middle=no_boseos_middle,
+        chunk_length=pipe.tokenizer.model_max_length,
+    )
+    prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device=pipe.device)
+    if uncond_prompt is not None:
+        uncond_tokens, uncond_weights = pad_tokens_and_weights(
+            uncond_tokens,
+            uncond_weights,
+            max_length,
+            bos,
+            eos,
+            pad,
+            no_boseos_middle=no_boseos_middle,
+            chunk_length=pipe.tokenizer.model_max_length,
+        )
+        uncond_tokens = torch.tensor(uncond_tokens, dtype=torch.long, device=pipe.device)
+
+    # get the embeddings
+    text_embeddings, text_pool = get_unweighted_text_embeddings(
+        pipe,
+        prompt_tokens,
+        pipe.tokenizer.model_max_length,
+        clip_skip,
+        eos,
+        pad,
+        is_sdxl_text_encoder2,
+        no_boseos_middle=no_boseos_middle,
+    )
+    prompt_weights = torch.tensor(prompt_weights, dtype=text_embeddings.dtype, device=pipe.device)
+
+    if uncond_prompt is not None:
+        uncond_embeddings, uncond_pool = get_unweighted_text_embeddings(
+            pipe,
+            uncond_tokens,
+            pipe.tokenizer.model_max_length,
+            clip_skip,
+            eos,
+            pad,
+            is_sdxl_text_encoder2,
+            no_boseos_middle=no_boseos_middle,
+        )
+        uncond_weights = torch.tensor(uncond_weights, dtype=uncond_embeddings.dtype, device=pipe.device)
+
+    # assign weights to the prompts and normalize in the sense of mean
+    # TODO: should we normalize by chunk or in a whole (current implementation)?
+    if (not skip_parsing) and (not skip_weighting):
+        previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+        text_embeddings *= prompt_weights.unsqueeze(-1)
+        current_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+        text_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)
+        if uncond_prompt is not None:
+            previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)
+            uncond_embeddings *= uncond_weights.unsqueeze(-1)
+            current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)
+            uncond_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)
+
+    if uncond_prompt is not None:
+        return text_embeddings, text_pool, uncond_embeddings, uncond_pool
+    return text_embeddings, text_pool, None, None
+
+
+def preprocess_image(image):
+    w, h = image.size
+    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32
+    image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"])
+    image = np.array(image).astype(np.float32) / 255.0
+    image = image[None].transpose(0, 3, 1, 2)
+    image = torch.from_numpy(image)
+    return 2.0 * image - 1.0
+
+
+def preprocess_mask(mask, scale_factor=8):
+    mask = mask.convert("L")
+    w, h = mask.size
+    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32
+    mask = mask.resize((w // scale_factor, h // scale_factor), resample=PIL_INTERPOLATION["nearest"])
+    mask = np.array(mask).astype(np.float32) / 255.0
+    mask = np.tile(mask, (4, 1, 1))
+    mask = mask[None].transpose(0, 1, 2, 3)  # what does this step do?
+    mask = 1 - mask  # repaint white, keep black
+    mask = torch.from_numpy(mask)
+    return mask
+
+
+def prepare_controlnet_image(
+    image: PIL.Image.Image,
+    width: int,
+    height: int,
+    batch_size: int,
+    num_images_per_prompt: int,
+    device: torch.device,
+    dtype: torch.dtype,
+    do_classifier_free_guidance: bool = False,
+    guess_mode: bool = False,
+):
+    if not isinstance(image, torch.Tensor):
+        if isinstance(image, PIL.Image.Image):
+            image = [image]
+
+        if isinstance(image[0], PIL.Image.Image):
+            images = []
+
+            for image_ in image:
+                image_ = image_.convert("RGB")
+                image_ = image_.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])
+                image_ = np.array(image_)
+                image_ = image_[None, :]
+                images.append(image_)
+
+            image = images
+
+            image = np.concatenate(image, axis=0)
+            image = np.array(image).astype(np.float32) / 255.0
+            image = image.transpose(0, 3, 1, 2)
+            image = torch.from_numpy(image)
+        elif isinstance(image[0], torch.Tensor):
+            image = torch.cat(image, dim=0)
+
+    image_batch_size = image.shape[0]
+
+    if image_batch_size == 1:
+        repeat_by = batch_size
+    else:
+        # image batch size is the same as prompt batch size
+        repeat_by = num_images_per_prompt
+
+    image = image.repeat_interleave(repeat_by, dim=0)
+
+    image = image.to(device=device, dtype=dtype)
+
+    if do_classifier_free_guidance and not guess_mode:
+        image = torch.cat([image] * 2)
+
+    return image
+
+
+class SdxlStableDiffusionLongPromptWeightingPipeline:
+    r"""
+    Pipeline for text-to-image generation using Stable Diffusion without tokens length limit, and support parsing
+    weighting in prompt.
+
+    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
+    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
+
+    Args:
+        vae ([`AutoencoderKL`]):
+            Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
+        text_encoder ([`CLIPTextModel`]):
+            Frozen text-encoder. Stable Diffusion uses the text portion of
+            [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
+            the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
+        tokenizer (`CLIPTokenizer`):
+            Tokenizer of class
+            [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
+        unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
+        scheduler ([`SchedulerMixin`]):
+            A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
+            [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
+        safety_checker ([`StableDiffusionSafetyChecker`]):
+            Classification module that estimates whether generated images could be considered offensive or harmful.
+            Please, refer to the [model card](https://huggingface.co/CompVis/stable-diffusion-v1-4) for details.
+        feature_extractor ([`CLIPFeatureExtractor`]):
+            Model that extracts features from generated images to be used as inputs for the `safety_checker`.
+    """
+
+    # if version.parse(version.parse(diffusers.__version__).base_version) >= version.parse("0.9.0"):
+
+    def __init__(
+        self,
+        vae: AutoencoderKL,
+        text_encoder: List[CLIPTextModel],
+        tokenizer: List[CLIPTokenizer],
+        unet: UNet2DConditionModel,
+        scheduler: SchedulerMixin,
+        # clip_skip: int,
+        safety_checker: StableDiffusionSafetyChecker,
+        feature_extractor: CLIPFeatureExtractor,
+        requires_safety_checker: bool = True,
+        clip_skip: int = 1,
+    ):
+        # clip skip is ignored currently
+        self.tokenizer = tokenizer[0]
+        self.text_encoder = text_encoder[0]
+        self.unet = unet
+        self.scheduler = scheduler
+        self.safety_checker = safety_checker
+        self.feature_extractor = feature_extractor
+        self.requires_safety_checker = requires_safety_checker
+        self.vae = vae
+        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
+        self.progress_bar = lambda x: tqdm(x, leave=False)
+
+        self.clip_skip = clip_skip
+        self.tokenizers = tokenizer
+        self.text_encoders = text_encoder
+
+    #     self.__init__additional__()
+
+    # def __init__additional__(self):
+    #     if not hasattr(self, "vae_scale_factor"):
+    #         setattr(self, "vae_scale_factor", 2 ** (len(self.vae.config.block_out_channels) - 1))
+
+    def to(self, device=None, dtype=None):
+        if device is not None:
+            self.device = device
+            # self.vae.to(device=self.device)
+        if dtype is not None:
+            self.dtype = dtype
+
+        # do not move Text Encoders to device, because Text Encoder should be on CPU
+
+    @property
+    def _execution_device(self):
+        r"""
+        Returns the device on which the pipeline's models will be executed. After calling
+        `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module
+        hooks.
+        """
+        if self.device != torch.device("meta") or not hasattr(self.unet, "_hf_hook"):
+            return self.device
+        for module in self.unet.modules():
+            if (
+                hasattr(module, "_hf_hook")
+                and hasattr(module._hf_hook, "execution_device")
+                and module._hf_hook.execution_device is not None
+            ):
+                return torch.device(module._hf_hook.execution_device)
+        return self.device
+
+    def _encode_prompt(
+        self,
+        prompt,
+        device,
+        num_images_per_prompt,
+        do_classifier_free_guidance,
+        negative_prompt,
+        max_embeddings_multiples,
+        is_sdxl_text_encoder2,
+    ):
+        r"""
+        Encodes the prompt into text encoder hidden states.
+
+        Args:
+            prompt (`str` or `list(int)`):
+                prompt to be encoded
+            device: (`torch.device`):
+                torch device
+            num_images_per_prompt (`int`):
+                number of images that should be generated per prompt
+            do_classifier_free_guidance (`bool`):
+                whether to use classifier free guidance or not
+            negative_prompt (`str` or `List[str]`):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+        """
+        batch_size = len(prompt) if isinstance(prompt, list) else 1
+
+        if negative_prompt is None:
+            negative_prompt = [""] * batch_size
+        elif isinstance(negative_prompt, str):
+            negative_prompt = [negative_prompt] * batch_size
+        if batch_size != len(negative_prompt):
+            raise ValueError(
+                f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
+                f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
+                " the batch size of `prompt`."
+            )
+
+        text_embeddings, text_pool, uncond_embeddings, uncond_pool = get_weighted_text_embeddings(
+            pipe=self,
+            prompt=prompt,
+            uncond_prompt=negative_prompt if do_classifier_free_guidance else None,
+            max_embeddings_multiples=max_embeddings_multiples,
+            clip_skip=self.clip_skip,
+            is_sdxl_text_encoder2=is_sdxl_text_encoder2,
+        )
+        bs_embed, seq_len, _ = text_embeddings.shape
+        text_embeddings = text_embeddings.repeat(1, num_images_per_prompt, 1)  # ??
+        text_embeddings = text_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)
+        if text_pool is not None:
+            text_pool = text_pool.repeat(1, num_images_per_prompt)
+            text_pool = text_pool.view(bs_embed * num_images_per_prompt, -1)
+
+        if do_classifier_free_guidance:
+            bs_embed, seq_len, _ = uncond_embeddings.shape
+            uncond_embeddings = uncond_embeddings.repeat(1, num_images_per_prompt, 1)
+            uncond_embeddings = uncond_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)
+            if uncond_pool is not None:
+                uncond_pool = uncond_pool.repeat(1, num_images_per_prompt)
+                uncond_pool = uncond_pool.view(bs_embed * num_images_per_prompt, -1)
+
+            return text_embeddings, text_pool, uncond_embeddings, uncond_pool
+
+        return text_embeddings, text_pool, None, None
+
+    def check_inputs(self, prompt, height, width, strength, callback_steps):
+        if not isinstance(prompt, str) and not isinstance(prompt, list):
+            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
+
+        if strength < 0 or strength > 1:
+            raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
+
+        if height % 8 != 0 or width % 8 != 0:
+            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
+
+        if (callback_steps is None) or (
+            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
+        ):
+            raise ValueError(
+                f"`callback_steps` has to be a positive integer but is {callback_steps} of type" f" {type(callback_steps)}."
+            )
+
+    def get_timesteps(self, num_inference_steps, strength, device, is_text2img):
+        if is_text2img:
+            return self.scheduler.timesteps.to(device), num_inference_steps
+        else:
+            # get the original timestep using init_timestep
+            offset = self.scheduler.config.get("steps_offset", 0)
+            init_timestep = int(num_inference_steps * strength) + offset
+            init_timestep = min(init_timestep, num_inference_steps)
+
+            t_start = max(num_inference_steps - init_timestep + offset, 0)
+            timesteps = self.scheduler.timesteps[t_start:].to(device)
+            return timesteps, num_inference_steps - t_start
+
+    def run_safety_checker(self, image, device, dtype):
+        if self.safety_checker is not None:
+            safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)
+            image, has_nsfw_concept = self.safety_checker(images=image, clip_input=safety_checker_input.pixel_values.to(dtype))
+        else:
+            has_nsfw_concept = None
+        return image, has_nsfw_concept
+
+    def decode_latents(self, latents):
+        with torch.no_grad():
+            latents = 1 / sdxl_model_util.VAE_SCALE_FACTOR * latents
+
+            # print("post_quant_conv dtype:", self.vae.post_quant_conv.weight.dtype)  # torch.float32
+            # x = torch.nn.functional.conv2d(latents, self.vae.post_quant_conv.weight.detach(), stride=1, padding=0)
+            # print("latents dtype:", latents.dtype, "x dtype:", x.dtype)  # torch.float32, torch.float16
+            # self.vae.to("cpu")
+            # self.vae.set_use_memory_efficient_attention_xformers(False)
+            # image = self.vae.decode(latents.to("cpu")).sample
+
+            image = self.vae.decode(latents.to(self.vae.dtype)).sample
+            image = (image / 2 + 0.5).clamp(0, 1)
+            # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
+            image = image.cpu().permute(0, 2, 3, 1).float().numpy()
+            return image
+
+    def prepare_extra_step_kwargs(self, generator, eta):
+        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
+        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
+        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
+        # and should be between [0, 1]
+
+        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
+        extra_step_kwargs = {}
+        if accepts_eta:
+            extra_step_kwargs["eta"] = eta
+
+        # check if the scheduler accepts generator
+        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
+        if accepts_generator:
+            extra_step_kwargs["generator"] = generator
+        return extra_step_kwargs
+
+    def prepare_latents(self, image, timestep, batch_size, height, width, dtype, device, generator, latents=None):
+        if image is None:
+            shape = (
+                batch_size,
+                self.unet.in_channels,
+                height // self.vae_scale_factor,
+                width // self.vae_scale_factor,
+            )
+
+            if latents is None:
+                if device.type == "mps":
+                    # randn does not work reproducibly on mps
+                    latents = torch.randn(shape, generator=generator, device="cpu", dtype=dtype).to(device)
+                else:
+                    latents = torch.randn(shape, generator=generator, device=device, dtype=dtype)
+            else:
+                if latents.shape != shape:
+                    raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}")
+                latents = latents.to(device)
+
+            # scale the initial noise by the standard deviation required by the scheduler
+            latents = latents * self.scheduler.init_noise_sigma
+            return latents, None, None
+        else:
+            init_latent_dist = self.vae.encode(image).latent_dist
+            init_latents = init_latent_dist.sample(generator=generator)
+            init_latents = sdxl_model_util.VAE_SCALE_FACTOR * init_latents
+            init_latents = torch.cat([init_latents] * batch_size, dim=0)
+            init_latents_orig = init_latents
+            shape = init_latents.shape
+
+            # add noise to latents using the timesteps
+            if device.type == "mps":
+                noise = torch.randn(shape, generator=generator, device="cpu", dtype=dtype).to(device)
+            else:
+                noise = torch.randn(shape, generator=generator, device=device, dtype=dtype)
+            latents = self.scheduler.add_noise(init_latents, noise, timestep)
+            return latents, init_latents_orig, noise
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        image: Union[torch.FloatTensor, PIL.Image.Image] = None,
+        mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None,
+        height: int = 512,
+        width: int = 512,
+        num_inference_steps: int = 50,
+        guidance_scale: float = 7.5,
+        strength: float = 0.8,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[torch.Generator] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        controlnet=None,
+        controlnet_image=None,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: int = 1,
+    ):
+        r"""
+        Function invoked when calling the pipeline for generation.
+
+        Args:
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process.
+            mask_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be
+                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a
+                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should
+                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.
+            height (`int`, *optional*, defaults to 512):
+                The height in pixels of the generated image.
+            width (`int`, *optional*, defaults to 512):
+                The width in pixels of the generated image.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.
+                `image` will be used as a starting point, adding more noise to it the larger the `strength`. The
+                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added
+                noise will be maximum and the denoising process will run for the full number of iterations specified in
+                `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            controlnet (`diffusers.ControlNetModel`, *optional*):
+                A controlnet model to be used for the inference. If not provided, controlnet will be disabled.
+            controlnet_image (`torch.FloatTensor` or `PIL.Image.Image`, *optional*):
+                `Image`, or tensor representing an image batch, to be used as the starting point for the controlnet
+                inference.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            is_cancelled_callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. If the function returns
+                `True`, the inference will be cancelled.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+
+        Returns:
+            `None` if cancelled by `is_cancelled_callback`,
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        if controlnet is not None and controlnet_image is None:
+            raise ValueError("controlnet_image must be provided if controlnet is not None.")
+
+        # 0. Default height and width to unet
+        height = height or self.unet.config.sample_size * self.vae_scale_factor
+        width = width or self.unet.config.sample_size * self.vae_scale_factor
+
+        # 1. Check inputs. Raise error if not correct
+        self.check_inputs(prompt, height, width, strength, callback_steps)
+
+        # 2. Define call parameters
+        batch_size = 1 if isinstance(prompt, str) else len(prompt)
+        device = self._execution_device
+        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
+        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
+        # corresponds to doing no classifier free guidance.
+        do_classifier_free_guidance = guidance_scale > 1.0
+
+        # 3. Encode input prompt
+        # 実装を簡単にするためにtokenzer/text encoderを切り替えて二回呼び出す
+        # To simplify the implementation, switch the tokenzer/text encoder and call it twice
+        text_embeddings_list = []
+        text_pool = None
+        uncond_embeddings_list = []
+        uncond_pool = None
+        for i in range(len(self.tokenizers)):
+            self.tokenizer = self.tokenizers[i]
+            self.text_encoder = self.text_encoders[i]
+
+            text_embeddings, tp1, uncond_embeddings, up1 = self._encode_prompt(
+                prompt,
+                device,
+                num_images_per_prompt,
+                do_classifier_free_guidance,
+                negative_prompt,
+                max_embeddings_multiples,
+                is_sdxl_text_encoder2=i == 1,
+            )
+            text_embeddings_list.append(text_embeddings)
+            uncond_embeddings_list.append(uncond_embeddings)
+
+            if tp1 is not None:
+                text_pool = tp1
+            if up1 is not None:
+                uncond_pool = up1
+
+        dtype = self.unet.dtype
+
+        # 4. Preprocess image and mask
+        if isinstance(image, PIL.Image.Image):
+            image = preprocess_image(image)
+        if image is not None:
+            image = image.to(device=self.device, dtype=dtype)
+        if isinstance(mask_image, PIL.Image.Image):
+            mask_image = preprocess_mask(mask_image, self.vae_scale_factor)
+        if mask_image is not None:
+            mask = mask_image.to(device=self.device, dtype=dtype)
+            mask = torch.cat([mask] * batch_size * num_images_per_prompt)
+        else:
+            mask = None
+
+        # ControlNet is not working yet in SDXL, but keep the code here for future use
+        if controlnet_image is not None:
+            controlnet_image = prepare_controlnet_image(
+                controlnet_image, width, height, batch_size, 1, self.device, controlnet.dtype, do_classifier_free_guidance, False
+            )
+
+        # 5. set timesteps
+        self.scheduler.set_timesteps(num_inference_steps, device=device)
+        timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device, image is None)
+        latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
+
+        # 6. Prepare latent variables
+        latents, init_latents_orig, noise = self.prepare_latents(
+            image,
+            latent_timestep,
+            batch_size * num_images_per_prompt,
+            height,
+            width,
+            dtype,
+            device,
+            generator,
+            latents,
+        )
+
+        # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+        # create size embs and concat embeddings for SDXL
+        orig_size = torch.tensor([height, width]).repeat(batch_size * num_images_per_prompt, 1).to(dtype)
+        crop_size = torch.zeros_like(orig_size)
+        target_size = orig_size
+        embs = sdxl_train_util.get_size_embeddings(orig_size, crop_size, target_size, device).to(dtype)
+
+        # make conditionings
+        if do_classifier_free_guidance:
+            text_embeddings = torch.cat(text_embeddings_list, dim=2)
+            uncond_embeddings = torch.cat(uncond_embeddings_list, dim=2)
+            text_embedding = torch.cat([uncond_embeddings, text_embeddings]).to(dtype)
+
+            cond_vector = torch.cat([text_pool, embs], dim=1)
+            uncond_vector = torch.cat([uncond_pool, embs], dim=1)
+            vector_embedding = torch.cat([uncond_vector, cond_vector]).to(dtype)
+        else:
+            text_embedding = torch.cat(text_embeddings_list, dim=2).to(dtype)
+            vector_embedding = torch.cat([text_pool, embs], dim=1).to(dtype)
+
+        # 8. Denoising loop
+        for i, t in enumerate(self.progress_bar(timesteps)):
+            # expand the latents if we are doing classifier free guidance
+            latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
+            latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+            unet_additional_args = {}
+            if controlnet is not None:
+                down_block_res_samples, mid_block_res_sample = controlnet(
+                    latent_model_input,
+                    t,
+                    encoder_hidden_states=text_embeddings,
+                    controlnet_cond=controlnet_image,
+                    conditioning_scale=1.0,
+                    guess_mode=False,
+                    return_dict=False,
+                )
+                unet_additional_args["down_block_additional_residuals"] = down_block_res_samples
+                unet_additional_args["mid_block_additional_residual"] = mid_block_res_sample
+
+            # predict the noise residual
+            noise_pred = self.unet(latent_model_input, t, text_embedding, vector_embedding)
+
+            # perform guidance
+            if do_classifier_free_guidance:
+                noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+            # compute the previous noisy sample x_t -> x_t-1
+            latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
+
+            if mask is not None:
+                # masking
+                init_latents_proper = self.scheduler.add_noise(init_latents_orig, noise, torch.tensor([t]))
+                latents = (init_latents_proper * mask) + (latents * (1 - mask))
+
+            # call the callback, if provided
+            if i % callback_steps == 0:
+                if callback is not None:
+                    callback(i, t, latents)
+                if is_cancelled_callback is not None and is_cancelled_callback():
+                    return None
+
+        # 9. Post-processing
+        image = self.decode_latents(latents.to(torch.float32))
+
+        # 10. Run safety checker
+        image, has_nsfw_concept = image, None  #  self.run_safety_checker(image, device, text_embeddings.dtype)
+
+        # 11. Convert to PIL
+        if output_type == "pil":
+            image = self.numpy_to_pil(image)
+
+        if not return_dict:
+            return image, has_nsfw_concept
+
+        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)
+
+    # copy from pil_utils.py
+    def numpy_to_pil(self, images: np.ndarray) -> Image.Image:
+        """
+        Convert a numpy image or a batch of images to a PIL image.
+        """
+        if images.ndim == 3:
+            images = images[None, ...]
+        images = (images * 255).round().astype("uint8")
+        if images.shape[-1] == 1:
+            # special case for grayscale (single channel) images
+            pil_images = [Image.fromarray(image.squeeze(), mode="L") for image in images]
+        else:
+            pil_images = [Image.fromarray(image) for image in images]
+
+        return pil_images
+
+    def text2img(
+        self,
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        height: int = 512,
+        width: int = 512,
+        num_inference_steps: int = 50,
+        guidance_scale: float = 7.5,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[torch.Generator] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: int = 1,
+    ):
+        r"""
+        Function for text-to-image generation.
+        Args:
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            height (`int`, *optional*, defaults to 512):
+                The height in pixels of the generated image.
+            width (`int`, *optional*, defaults to 512):
+                The width in pixels of the generated image.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            latents (`torch.FloatTensor`, *optional*):
+                Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+                generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+                tensor will ge generated by sampling using the supplied random `generator`.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            is_cancelled_callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. If the function returns
+                `True`, the inference will be cancelled.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        return self.__call__(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            height=height,
+            width=width,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            num_images_per_prompt=num_images_per_prompt,
+            eta=eta,
+            generator=generator,
+            latents=latents,
+            max_embeddings_multiples=max_embeddings_multiples,
+            output_type=output_type,
+            return_dict=return_dict,
+            callback=callback,
+            is_cancelled_callback=is_cancelled_callback,
+            callback_steps=callback_steps,
+        )
+
+    def img2img(
+        self,
+        image: Union[torch.FloatTensor, PIL.Image.Image],
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        strength: float = 0.8,
+        num_inference_steps: Optional[int] = 50,
+        guidance_scale: Optional[float] = 7.5,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: Optional[float] = 0.0,
+        generator: Optional[torch.Generator] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: int = 1,
+    ):
+        r"""
+        Function for image-to-image generation.
+        Args:
+            image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process.
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1.
+                `image` will be used as a starting point, adding more noise to it the larger the `strength`. The
+                number of denoising steps depends on the amount of noise initially added. When `strength` is 1, added
+                noise will be maximum and the denoising process will run for the full number of iterations specified in
+                `num_inference_steps`. A value of 1, therefore, essentially ignores `image`.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference. This parameter will be modulated by `strength`.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            is_cancelled_callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. If the function returns
+                `True`, the inference will be cancelled.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        return self.__call__(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image=image,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            strength=strength,
+            num_images_per_prompt=num_images_per_prompt,
+            eta=eta,
+            generator=generator,
+            max_embeddings_multiples=max_embeddings_multiples,
+            output_type=output_type,
+            return_dict=return_dict,
+            callback=callback,
+            is_cancelled_callback=is_cancelled_callback,
+            callback_steps=callback_steps,
+        )
+
+    def inpaint(
+        self,
+        image: Union[torch.FloatTensor, PIL.Image.Image],
+        mask_image: Union[torch.FloatTensor, PIL.Image.Image],
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        strength: float = 0.8,
+        num_inference_steps: Optional[int] = 50,
+        guidance_scale: Optional[float] = 7.5,
+        num_images_per_prompt: Optional[int] = 1,
+        eta: Optional[float] = 0.0,
+        generator: Optional[torch.Generator] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: int = 1,
+    ):
+        r"""
+        Function for inpaint.
+        Args:
+            image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process. This is the image whose masked region will be inpainted.
+            mask_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be
+                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a
+                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should
+                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to inpaint the masked area. Must be between 0 and 1. When `strength`
+                is 1, the denoising process will be run on the masked area for the full number of iterations specified
+                in `num_inference_steps`. `image` will be used as a reference for the masked area, adding more
+                noise to that region the larger the `strength`. If `strength` is 0, no inpainting will occur.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The reference number of denoising steps. More denoising steps usually lead to a higher quality image at
+                the expense of slower inference. This parameter will be modulated by `strength`, as explained above.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            max_embeddings_multiples (`int`, *optional*, defaults to `3`):
+                The max multiple length of prompt embeddings compared to the max output length of text encoder.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            is_cancelled_callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. If the function returns
+                `True`, the inference will be cancelled.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        return self.__call__(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image=image,
+            mask_image=mask_image,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            strength=strength,
+            num_images_per_prompt=num_images_per_prompt,
+            eta=eta,
+            generator=generator,
+            max_embeddings_multiples=max_embeddings_multiples,
+            output_type=output_type,
+            return_dict=return_dict,
+            callback=callback,
+            is_cancelled_callback=is_cancelled_callback,
+            callback_steps=callback_steps,
+        )
diff --git a/library/sdxl_model_util.py b/library/sdxl_model_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae764b17f7c282b7e3ff35a727659e96524f6c74
--- /dev/null
+++ b/library/sdxl_model_util.py
@@ -0,0 +1,316 @@
+import torch
+from safetensors.torch import load_file, save_file
+from transformers import CLIPTextModel, CLIPTextConfig, CLIPTextModelWithProjection
+from diffusers import AutoencoderKL
+from library import model_util
+from library import sdxl_original_unet
+
+
+VAE_SCALE_FACTOR = 0.13025
+MODEL_VERSION_SDXL_BASE_V0_9 = "sdxl_base_v0-9"
+
+
+def convert_sdxl_text_encoder_2_checkpoint(checkpoint, max_length):
+    SDXL_KEY_PREFIX = "conditioner.embedders.1.model."
+
+    # SD2のと、基本的には同じ。logit_scaleを後で使うので、それを追加で返す
+    # logit_scaleはcheckpointの保存時に使用する
+    def convert_key(key):
+        # common conversion
+        key = key.replace(SDXL_KEY_PREFIX + "transformer.", "text_model.encoder.")
+        key = key.replace(SDXL_KEY_PREFIX, "text_model.")
+
+        if "resblocks" in key:
+            # resblocks conversion
+            key = key.replace(".resblocks.", ".layers.")
+            if ".ln_" in key:
+                key = key.replace(".ln_", ".layer_norm")
+            elif ".mlp." in key:
+                key = key.replace(".c_fc.", ".fc1.")
+                key = key.replace(".c_proj.", ".fc2.")
+            elif ".attn.out_proj" in key:
+                key = key.replace(".attn.out_proj.", ".self_attn.out_proj.")
+            elif ".attn.in_proj" in key:
+                key = None  # 特殊なので後で処理する
+            else:
+                raise ValueError(f"unexpected key in SD: {key}")
+        elif ".positional_embedding" in key:
+            key = key.replace(".positional_embedding", ".embeddings.position_embedding.weight")
+        elif ".text_projection" in key:
+            key = key.replace("text_model.text_projection", "text_projection.weight")
+        elif ".logit_scale" in key:
+            key = None  # 後で処理する
+        elif ".token_embedding" in key:
+            key = key.replace(".token_embedding.weight", ".embeddings.token_embedding.weight")
+        elif ".ln_final" in key:
+            key = key.replace(".ln_final", ".final_layer_norm")
+        # ckpt from comfy has this key: text_model.encoder.text_model.embeddings.position_ids
+        elif ".embeddings.position_ids" in key:
+            key = None  # remove this key: make position_ids by ourselves
+        return key
+
+    keys = list(checkpoint.keys())
+    new_sd = {}
+    for key in keys:
+        new_key = convert_key(key)
+        if new_key is None:
+            continue
+        new_sd[new_key] = checkpoint[key]
+
+    # attnの変換
+    for key in keys:
+        if ".resblocks" in key and ".attn.in_proj_" in key:
+            # 三つに分割
+            values = torch.chunk(checkpoint[key], 3)
+
+            key_suffix = ".weight" if "weight" in key else ".bias"
+            key_pfx = key.replace(SDXL_KEY_PREFIX + "transformer.resblocks.", "text_model.encoder.layers.")
+            key_pfx = key_pfx.replace("_weight", "")
+            key_pfx = key_pfx.replace("_bias", "")
+            key_pfx = key_pfx.replace(".attn.in_proj", ".self_attn.")
+            new_sd[key_pfx + "q_proj" + key_suffix] = values[0]
+            new_sd[key_pfx + "k_proj" + key_suffix] = values[1]
+            new_sd[key_pfx + "v_proj" + key_suffix] = values[2]
+
+    # original SD にはないので、position_idsを追加
+    position_ids = torch.Tensor([list(range(max_length))]).to(torch.int64)
+    new_sd["text_model.embeddings.position_ids"] = position_ids
+
+    # logit_scale はDiffusersには含まれないが、保存時に戻したいので別途返す
+    logit_scale = checkpoint.get(SDXL_KEY_PREFIX + "logit_scale", None)
+
+    return new_sd, logit_scale
+
+
+def load_models_from_sdxl_checkpoint(model_version, ckpt_path, map_location):
+    # model_version is reserved for future use
+
+    # Load the state dict
+    if model_util.is_safetensors(ckpt_path):
+        checkpoint = None
+        state_dict = load_file(ckpt_path, device=map_location)
+        epoch = None
+        global_step = None
+    else:
+        checkpoint = torch.load(ckpt_path, map_location=map_location)
+        if "state_dict" in checkpoint:
+            state_dict = checkpoint["state_dict"]
+            epoch = checkpoint.get("epoch", 0)
+            global_step = checkpoint.get("global_step", 0)
+        else:
+            state_dict = checkpoint
+            epoch = 0
+            global_step = 0
+        checkpoint = None
+
+    # U-Net
+    print("building U-Net")
+    unet = sdxl_original_unet.SdxlUNet2DConditionModel()
+
+    print("loading U-Net from checkpoint")
+    unet_sd = {}
+    for k in list(state_dict.keys()):
+        if k.startswith("model.diffusion_model."):
+            unet_sd[k.replace("model.diffusion_model.", "")] = state_dict.pop(k)
+    info = unet.load_state_dict(unet_sd)
+    print("U-Net: ", info)
+    del unet_sd
+
+    # Text Encoders
+    print("building text encoders")
+
+    # Text Encoder 1 is same to SDXL
+    text_model1_cfg = CLIPTextConfig(
+        vocab_size=49408,
+        hidden_size=768,
+        intermediate_size=3072,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        max_position_embeddings=77,
+        hidden_act="quick_gelu",
+        layer_norm_eps=1e-05,
+        dropout=0.0,
+        attention_dropout=0.0,
+        initializer_range=0.02,
+        initializer_factor=1.0,
+        pad_token_id=1,
+        bos_token_id=0,
+        eos_token_id=2,
+        model_type="clip_text_model",
+        projection_dim=768,
+        # torch_dtype="float32",
+        # transformers_version="4.25.0.dev0",
+    )
+    text_model1 = CLIPTextModel._from_config(text_model1_cfg)
+
+    # Text Encoder 2 is different from SDXL. SDXL uses open clip, but we use the model from HuggingFace.
+    # Note: Tokenizer from HuggingFace is different from SDXL. We must use open clip's tokenizer.
+    text_model2_cfg = CLIPTextConfig(
+        vocab_size=49408,
+        hidden_size=1280,
+        intermediate_size=5120,
+        num_hidden_layers=32,
+        num_attention_heads=20,
+        max_position_embeddings=77,
+        hidden_act="gelu",
+        layer_norm_eps=1e-05,
+        dropout=0.0,
+        attention_dropout=0.0,
+        initializer_range=0.02,
+        initializer_factor=1.0,
+        pad_token_id=1,
+        bos_token_id=0,
+        eos_token_id=2,
+        model_type="clip_text_model",
+        projection_dim=1280,
+        # torch_dtype="float32",
+        # transformers_version="4.25.0.dev0",
+    )
+    text_model2 = CLIPTextModelWithProjection(text_model2_cfg)
+
+    print("loading text encoders from checkpoint")
+    te1_sd = {}
+    te2_sd = {}
+    for k in list(state_dict.keys()):
+        if k.startswith("conditioner.embedders.0.transformer."):
+            te1_sd[k.replace("conditioner.embedders.0.transformer.", "")] = state_dict.pop(k)
+        elif k.startswith("conditioner.embedders.1.model."):
+            te2_sd[k] = state_dict.pop(k)
+
+    info1 = text_model1.load_state_dict(te1_sd)
+    print("text encoder 1:", info1)
+
+    converted_sd, logit_scale = convert_sdxl_text_encoder_2_checkpoint(te2_sd, max_length=77)
+    info2 = text_model2.load_state_dict(converted_sd)
+    print("text encoder 2:", info2)
+
+    # prepare vae
+    print("building VAE")
+    vae_config = model_util.create_vae_diffusers_config()
+    vae = AutoencoderKL(**vae_config)  # .to(device)
+
+    print("loading VAE from checkpoint")
+    converted_vae_checkpoint = model_util.convert_ldm_vae_checkpoint(state_dict, vae_config)
+    info = vae.load_state_dict(converted_vae_checkpoint)
+    print("VAE:", info)
+
+    ckpt_info = (epoch, global_step) if epoch is not None else None
+    return text_model1, text_model2, vae, unet, logit_scale, ckpt_info
+
+
+def convert_text_encoder_2_state_dict_to_sdxl(checkpoint, logit_scale):
+    def convert_key(key):
+        # position_idsの除去
+        if ".position_ids" in key:
+            return None
+
+        # common
+        key = key.replace("text_model.encoder.", "transformer.")
+        key = key.replace("text_model.", "")
+        if "layers" in key:
+            # resblocks conversion
+            key = key.replace(".layers.", ".resblocks.")
+            if ".layer_norm" in key:
+                key = key.replace(".layer_norm", ".ln_")
+            elif ".mlp." in key:
+                key = key.replace(".fc1.", ".c_fc.")
+                key = key.replace(".fc2.", ".c_proj.")
+            elif ".self_attn.out_proj" in key:
+                key = key.replace(".self_attn.out_proj.", ".attn.out_proj.")
+            elif ".self_attn." in key:
+                key = None  # 特殊なので後で処理する
+            else:
+                raise ValueError(f"unexpected key in DiffUsers model: {key}")
+        elif ".position_embedding" in key:
+            key = key.replace("embeddings.position_embedding.weight", "positional_embedding")
+        elif ".token_embedding" in key:
+            key = key.replace("embeddings.token_embedding.weight", "token_embedding.weight")
+        elif "text_projection" in key:  # no dot in key
+            key = key.replace("text_projection.weight", "text_projection")
+        elif "final_layer_norm" in key:
+            key = key.replace("final_layer_norm", "ln_final")
+        return key
+
+    keys = list(checkpoint.keys())
+    new_sd = {}
+    for key in keys:
+        new_key = convert_key(key)
+        if new_key is None:
+            continue
+        new_sd[new_key] = checkpoint[key]
+
+    # attnの変換
+    for key in keys:
+        if "layers" in key and "q_proj" in key:
+            # 三つを結合
+            key_q = key
+            key_k = key.replace("q_proj", "k_proj")
+            key_v = key.replace("q_proj", "v_proj")
+
+            value_q = checkpoint[key_q]
+            value_k = checkpoint[key_k]
+            value_v = checkpoint[key_v]
+            value = torch.cat([value_q, value_k, value_v])
+
+            new_key = key.replace("text_model.encoder.layers.", "transformer.resblocks.")
+            new_key = new_key.replace(".self_attn.q_proj.", ".attn.in_proj_")
+            new_sd[new_key] = value
+
+    if logit_scale is not None:
+        new_sd["logit_scale"] = logit_scale
+
+    return new_sd
+
+
+def save_stable_diffusion_checkpoint(
+    output_file,
+    text_encoder1,
+    text_encoder2,
+    unet,
+    epochs,
+    steps,
+    ckpt_info,
+    vae,
+    logit_scale,
+    save_dtype=None,
+):
+    state_dict = {}
+
+    def update_sd(prefix, sd):
+        for k, v in sd.items():
+            key = prefix + k
+            if save_dtype is not None:
+                v = v.detach().clone().to("cpu").to(save_dtype)
+            state_dict[key] = v
+
+    # Convert the UNet model
+    update_sd("model.diffusion_model.", unet.state_dict())
+
+    # Convert the text encoders
+    update_sd("conditioner.embedders.0.transformer.", text_encoder1.state_dict())
+
+    text_enc2_dict = convert_text_encoder_2_state_dict_to_sdxl(text_encoder2.state_dict(), logit_scale)
+    update_sd("conditioner.embedders.1.model.", text_enc2_dict)
+
+    # Convert the VAE
+    vae_dict = model_util.convert_vae_state_dict(vae.state_dict())
+    update_sd("first_stage_model.", vae_dict)
+
+    # Put together new checkpoint
+    key_count = len(state_dict.keys())
+    new_ckpt = {"state_dict": state_dict}
+
+    # epoch and global_step are sometimes not int
+    if ckpt_info is not None:
+        epochs += ckpt_info[0]
+        steps += ckpt_info[1]
+
+    new_ckpt["epoch"] = epochs
+    new_ckpt["global_step"] = steps
+
+    if model_util.is_safetensors(output_file):
+        save_file(state_dict, output_file)
+    else:
+        torch.save(new_ckpt, output_file)
+
+    return key_count
diff --git a/library/sdxl_original_unet.py b/library/sdxl_original_unet.py
new file mode 100644
index 0000000000000000000000000000000000000000..8ba1c988b97616c5abf5fabc4716d126ca067e1d
--- /dev/null
+++ b/library/sdxl_original_unet.py
@@ -0,0 +1,1159 @@
+# Diffusersのコードをベースとした sd_xl_baseのU-Net
+# state dictの形式をSDXLに合わせてある
+
+"""
+      target: sgm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        adm_in_channels: 2816
+        num_classes: sequential
+        use_checkpoint: True
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [4, 2]
+        num_res_blocks: 2
+        channel_mult: [1, 2, 4]
+        num_head_channels: 64
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: [1, 2, 10]  # note: the first is unused (due to attn_res starting at 2) 32, 16, 8 --> 64, 32, 16
+        context_dim: 2048
+        spatial_transformer_attn_type: softmax-xformers
+        legacy: False
+"""
+
+import math
+from types import SimpleNamespace
+from typing import Optional
+import torch
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import functional as F
+from einops import rearrange
+
+
+IN_CHANNELS: int = 4
+OUT_CHANNELS: int = 4
+ADM_IN_CHANNELS: int = 2816
+CONTEXT_DIM: int = 2048
+MODEL_CHANNELS: int = 320
+TIME_EMBED_DIM = 320 * 4
+
+
+# region memory effcient attention
+
+# FlashAttentionを使うCrossAttention
+# based on https://github.com/lucidrains/memory-efficient-attention-pytorch/blob/main/memory_efficient_attention_pytorch/flash_attention.py
+# LICENSE MIT https://github.com/lucidrains/memory-efficient-attention-pytorch/blob/main/LICENSE
+
+# constants
+
+EPSILON = 1e-6
+
+# helper functions
+
+
+def exists(val):
+    return val is not None
+
+
+def default(val, d):
+    return val if exists(val) else d
+
+
+# flash attention forwards and backwards
+
+# https://arxiv.org/abs/2205.14135
+
+
+class FlashAttentionFunction(torch.autograd.Function):
+    @staticmethod
+    @torch.no_grad()
+    def forward(ctx, q, k, v, mask, causal, q_bucket_size, k_bucket_size):
+        """Algorithm 2 in the paper"""
+
+        device = q.device
+        dtype = q.dtype
+        max_neg_value = -torch.finfo(q.dtype).max
+        qk_len_diff = max(k.shape[-2] - q.shape[-2], 0)
+
+        o = torch.zeros_like(q)
+        all_row_sums = torch.zeros((*q.shape[:-1], 1), dtype=dtype, device=device)
+        all_row_maxes = torch.full((*q.shape[:-1], 1), max_neg_value, dtype=dtype, device=device)
+
+        scale = q.shape[-1] ** -0.5
+
+        if not exists(mask):
+            mask = (None,) * math.ceil(q.shape[-2] / q_bucket_size)
+        else:
+            mask = rearrange(mask, "b n -> b 1 1 n")
+            mask = mask.split(q_bucket_size, dim=-1)
+
+        row_splits = zip(
+            q.split(q_bucket_size, dim=-2),
+            o.split(q_bucket_size, dim=-2),
+            mask,
+            all_row_sums.split(q_bucket_size, dim=-2),
+            all_row_maxes.split(q_bucket_size, dim=-2),
+        )
+
+        for ind, (qc, oc, row_mask, row_sums, row_maxes) in enumerate(row_splits):
+            q_start_index = ind * q_bucket_size - qk_len_diff
+
+            col_splits = zip(
+                k.split(k_bucket_size, dim=-2),
+                v.split(k_bucket_size, dim=-2),
+            )
+
+            for k_ind, (kc, vc) in enumerate(col_splits):
+                k_start_index = k_ind * k_bucket_size
+
+                attn_weights = torch.einsum("... i d, ... j d -> ... i j", qc, kc) * scale
+
+                if exists(row_mask):
+                    attn_weights.masked_fill_(~row_mask, max_neg_value)
+
+                if causal and q_start_index < (k_start_index + k_bucket_size - 1):
+                    causal_mask = torch.ones((qc.shape[-2], kc.shape[-2]), dtype=torch.bool, device=device).triu(
+                        q_start_index - k_start_index + 1
+                    )
+                    attn_weights.masked_fill_(causal_mask, max_neg_value)
+
+                block_row_maxes = attn_weights.amax(dim=-1, keepdims=True)
+                attn_weights -= block_row_maxes
+                exp_weights = torch.exp(attn_weights)
+
+                if exists(row_mask):
+                    exp_weights.masked_fill_(~row_mask, 0.0)
+
+                block_row_sums = exp_weights.sum(dim=-1, keepdims=True).clamp(min=EPSILON)
+
+                new_row_maxes = torch.maximum(block_row_maxes, row_maxes)
+
+                exp_values = torch.einsum("... i j, ... j d -> ... i d", exp_weights, vc)
+
+                exp_row_max_diff = torch.exp(row_maxes - new_row_maxes)
+                exp_block_row_max_diff = torch.exp(block_row_maxes - new_row_maxes)
+
+                new_row_sums = exp_row_max_diff * row_sums + exp_block_row_max_diff * block_row_sums
+
+                oc.mul_((row_sums / new_row_sums) * exp_row_max_diff).add_((exp_block_row_max_diff / new_row_sums) * exp_values)
+
+                row_maxes.copy_(new_row_maxes)
+                row_sums.copy_(new_row_sums)
+
+        ctx.args = (causal, scale, mask, q_bucket_size, k_bucket_size)
+        ctx.save_for_backward(q, k, v, o, all_row_sums, all_row_maxes)
+
+        return o
+
+    @staticmethod
+    @torch.no_grad()
+    def backward(ctx, do):
+        """Algorithm 4 in the paper"""
+
+        causal, scale, mask, q_bucket_size, k_bucket_size = ctx.args
+        q, k, v, o, l, m = ctx.saved_tensors
+
+        device = q.device
+
+        max_neg_value = -torch.finfo(q.dtype).max
+        qk_len_diff = max(k.shape[-2] - q.shape[-2], 0)
+
+        dq = torch.zeros_like(q)
+        dk = torch.zeros_like(k)
+        dv = torch.zeros_like(v)
+
+        row_splits = zip(
+            q.split(q_bucket_size, dim=-2),
+            o.split(q_bucket_size, dim=-2),
+            do.split(q_bucket_size, dim=-2),
+            mask,
+            l.split(q_bucket_size, dim=-2),
+            m.split(q_bucket_size, dim=-2),
+            dq.split(q_bucket_size, dim=-2),
+        )
+
+        for ind, (qc, oc, doc, row_mask, lc, mc, dqc) in enumerate(row_splits):
+            q_start_index = ind * q_bucket_size - qk_len_diff
+
+            col_splits = zip(
+                k.split(k_bucket_size, dim=-2),
+                v.split(k_bucket_size, dim=-2),
+                dk.split(k_bucket_size, dim=-2),
+                dv.split(k_bucket_size, dim=-2),
+            )
+
+            for k_ind, (kc, vc, dkc, dvc) in enumerate(col_splits):
+                k_start_index = k_ind * k_bucket_size
+
+                attn_weights = torch.einsum("... i d, ... j d -> ... i j", qc, kc) * scale
+
+                if causal and q_start_index < (k_start_index + k_bucket_size - 1):
+                    causal_mask = torch.ones((qc.shape[-2], kc.shape[-2]), dtype=torch.bool, device=device).triu(
+                        q_start_index - k_start_index + 1
+                    )
+                    attn_weights.masked_fill_(causal_mask, max_neg_value)
+
+                exp_attn_weights = torch.exp(attn_weights - mc)
+
+                if exists(row_mask):
+                    exp_attn_weights.masked_fill_(~row_mask, 0.0)
+
+                p = exp_attn_weights / lc
+
+                dv_chunk = torch.einsum("... i j, ... i d -> ... j d", p, doc)
+                dp = torch.einsum("... i d, ... j d -> ... i j", doc, vc)
+
+                D = (doc * oc).sum(dim=-1, keepdims=True)
+                ds = p * scale * (dp - D)
+
+                dq_chunk = torch.einsum("... i j, ... j d -> ... i d", ds, kc)
+                dk_chunk = torch.einsum("... i j, ... i d -> ... j d", ds, qc)
+
+                dqc.add_(dq_chunk)
+                dkc.add_(dk_chunk)
+                dvc.add_(dv_chunk)
+
+        return dq, dk, dv, None, None, None, None
+
+
+# endregion
+
+
+def get_parameter_dtype(parameter: torch.nn.Module):
+    return next(parameter.parameters()).dtype
+
+
+def get_parameter_device(parameter: torch.nn.Module):
+    return next(parameter.parameters()).device
+
+
+def get_timestep_embedding(
+    timesteps: torch.Tensor,
+    embedding_dim: int,
+    downscale_freq_shift: float = 1,
+    scale: float = 1,
+    max_period: int = 10000,
+):
+    """
+    This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings.
+
+    :param timesteps: a 1-D Tensor of N indices, one per batch element.
+                      These may be fractional.
+    :param embedding_dim: the dimension of the output. :param max_period: controls the minimum frequency of the
+    embeddings. :return: an [N x dim] Tensor of positional embeddings.
+    """
+    assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"
+
+    half_dim = embedding_dim // 2
+    exponent = -math.log(max_period) * torch.arange(start=0, end=half_dim, dtype=torch.float32, device=timesteps.device)
+    exponent = exponent / (half_dim - downscale_freq_shift)
+
+    emb = torch.exp(exponent)
+    emb = timesteps[:, None].float() * emb[None, :]
+
+    # scale embeddings
+    emb = scale * emb
+
+    # concat sine and cosine embeddings: flipped from Diffusers original ver because always flip_sin_to_cos=True
+    emb = torch.cat([torch.cos(emb), torch.sin(emb)], dim=-1)
+
+    # zero pad
+    if embedding_dim % 2 == 1:
+        emb = torch.nn.functional.pad(emb, (0, 1, 0, 0))
+    return emb
+
+
+class GroupNorm32(nn.GroupNorm):
+    def forward(self, x):
+        if self.weight.dtype != torch.float32:
+            return super().forward(x)
+        return super().forward(x.float()).type(x.dtype)
+
+
+class ResnetBlock2D(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+
+        self.in_layers = nn.Sequential(
+            GroupNorm32(32, in_channels),
+            nn.SiLU(),
+            nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1),
+        )
+
+        self.emb_layers = nn.Sequential(nn.SiLU(), nn.Linear(TIME_EMBED_DIM, out_channels))
+
+        self.out_layers = nn.Sequential(
+            GroupNorm32(32, out_channels),
+            nn.SiLU(),
+            nn.Identity(),  # to make state_dict compatible with original model
+            nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1),
+        )
+
+        if in_channels != out_channels:
+            self.skip_connection = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+        else:
+            self.skip_connection = nn.Identity()
+
+        self.gradient_checkpointing = False
+
+    def forward_body(self, x, emb):
+        h = self.in_layers(x)
+        emb_out = self.emb_layers(emb).type(h.dtype)
+        h = h + emb_out[:, :, None, None]
+        h = self.out_layers(h)
+        x = self.skip_connection(x)
+        return x + h
+
+    def forward(self, x, emb):
+        if self.training and self.gradient_checkpointing:
+            # print("ResnetBlock2D: gradient_checkpointing")
+
+            def create_custom_forward(func):
+                def custom_forward(*inputs):
+                    return func(*inputs)
+
+                return custom_forward
+
+            x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.forward_body), x, emb)
+        else:
+            x = self.forward_body(x, emb)
+
+        return x
+
+
+class Downsample2D(nn.Module):
+    def __init__(self, channels, out_channels):
+        super().__init__()
+
+        self.channels = channels
+        self.out_channels = out_channels
+
+        self.op = nn.Conv2d(self.channels, self.out_channels, 3, stride=2, padding=1)
+
+        self.gradient_checkpointing = False
+
+    def forward_body(self, hidden_states):
+        assert hidden_states.shape[1] == self.channels
+        hidden_states = self.op(hidden_states)
+
+        return hidden_states
+
+    def forward(self, hidden_states):
+        if self.training and self.gradient_checkpointing:
+            # print("Downsample2D: gradient_checkpointing")
+
+            def create_custom_forward(func):
+                def custom_forward(*inputs):
+                    return func(*inputs)
+
+                return custom_forward
+
+            hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(self.forward_body), hidden_states)
+        else:
+            hidden_states = self.forward_body(hidden_states)
+
+        return hidden_states
+
+
+class CrossAttention(nn.Module):
+    def __init__(
+        self,
+        query_dim: int,
+        cross_attention_dim: Optional[int] = None,
+        heads: int = 8,
+        dim_head: int = 64,
+        upcast_attention: bool = False,
+    ):
+        super().__init__()
+        inner_dim = dim_head * heads
+        cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dim
+        self.upcast_attention = upcast_attention
+
+        self.scale = dim_head**-0.5
+        self.heads = heads
+
+        self.to_q = nn.Linear(query_dim, inner_dim, bias=False)
+        self.to_k = nn.Linear(cross_attention_dim, inner_dim, bias=False)
+        self.to_v = nn.Linear(cross_attention_dim, inner_dim, bias=False)
+
+        self.to_out = nn.ModuleList([])
+        self.to_out.append(nn.Linear(inner_dim, query_dim))
+        # no dropout here
+
+        self.use_memory_efficient_attention_xformers = False
+        self.use_memory_efficient_attention_mem_eff = False
+        self.use_sdpa = False
+
+    def set_use_memory_efficient_attention(self, xformers, mem_eff):
+        self.use_memory_efficient_attention_xformers = xformers
+        self.use_memory_efficient_attention_mem_eff = mem_eff
+
+    def set_use_sdpa(self, sdpa):
+        self.use_sdpa = sdpa
+
+    def reshape_heads_to_batch_dim(self, tensor):
+        batch_size, seq_len, dim = tensor.shape
+        head_size = self.heads
+        tensor = tensor.reshape(batch_size, seq_len, head_size, dim // head_size)
+        tensor = tensor.permute(0, 2, 1, 3).reshape(batch_size * head_size, seq_len, dim // head_size)
+        return tensor
+
+    def reshape_batch_dim_to_heads(self, tensor):
+        batch_size, seq_len, dim = tensor.shape
+        head_size = self.heads
+        tensor = tensor.reshape(batch_size // head_size, head_size, seq_len, dim)
+        tensor = tensor.permute(0, 2, 1, 3).reshape(batch_size // head_size, seq_len, dim * head_size)
+        return tensor
+
+    def forward(self, hidden_states, context=None, mask=None):
+        if self.use_memory_efficient_attention_xformers:
+            return self.forward_memory_efficient_xformers(hidden_states, context, mask)
+        if self.use_memory_efficient_attention_mem_eff:
+            return self.forward_memory_efficient_mem_eff(hidden_states, context, mask)
+        if self.use_sdpa:
+            return self.forward_sdpa(hidden_states, context, mask)
+
+        query = self.to_q(hidden_states)
+        context = context if context is not None else hidden_states
+        key = self.to_k(context)
+        value = self.to_v(context)
+
+        query = self.reshape_heads_to_batch_dim(query)
+        key = self.reshape_heads_to_batch_dim(key)
+        value = self.reshape_heads_to_batch_dim(value)
+
+        hidden_states = self._attention(query, key, value)
+
+        # linear proj
+        hidden_states = self.to_out[0](hidden_states)
+        # hidden_states = self.to_out[1](hidden_states)     # no dropout
+        return hidden_states
+
+    def _attention(self, query, key, value):
+        if self.upcast_attention:
+            query = query.float()
+            key = key.float()
+
+        attention_scores = torch.baddbmm(
+            torch.empty(query.shape[0], query.shape[1], key.shape[1], dtype=query.dtype, device=query.device),
+            query,
+            key.transpose(-1, -2),
+            beta=0,
+            alpha=self.scale,
+        )
+        attention_probs = attention_scores.softmax(dim=-1)
+
+        # cast back to the original dtype
+        attention_probs = attention_probs.to(value.dtype)
+
+        # compute attention output
+        hidden_states = torch.bmm(attention_probs, value)
+
+        # reshape hidden_states
+        hidden_states = self.reshape_batch_dim_to_heads(hidden_states)
+        return hidden_states
+
+    # TODO support Hypernetworks
+    def forward_memory_efficient_xformers(self, x, context=None, mask=None):
+        import xformers.ops
+
+        h = self.heads
+        q_in = self.to_q(x)
+        context = context if context is not None else x
+        context = context.to(x.dtype)
+        k_in = self.to_k(context)
+        v_in = self.to_v(context)
+
+        q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b n h d", h=h), (q_in, k_in, v_in))
+        del q_in, k_in, v_in
+
+        q = q.contiguous()
+        k = k.contiguous()
+        v = v.contiguous()
+        out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None)  # 最適なのを選んでくれる
+        del q, k, v
+
+        out = rearrange(out, "b n h d -> b n (h d)", h=h)
+
+        out = self.to_out[0](out)
+        return out
+
+    def forward_memory_efficient_mem_eff(self, x, context=None, mask=None):
+        flash_func = FlashAttentionFunction
+
+        q_bucket_size = 512
+        k_bucket_size = 1024
+
+        h = self.heads
+        q = self.to_q(x)
+        context = context if context is not None else x
+        context = context.to(x.dtype)
+        k = self.to_k(context)
+        v = self.to_v(context)
+        del context, x
+
+        q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=h), (q, k, v))
+
+        out = flash_func.apply(q, k, v, mask, False, q_bucket_size, k_bucket_size)
+
+        out = rearrange(out, "b h n d -> b n (h d)")
+
+        out = self.to_out[0](out)
+        return out
+
+    def forward_sdpa(self, x, context=None, mask=None):
+        h = self.heads
+        q_in = self.to_q(x)
+        context = context if context is not None else x
+        context = context.to(x.dtype)
+        k_in = self.to_k(context)
+        v_in = self.to_v(context)
+
+        q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=h), (q_in, k_in, v_in))
+        del q_in, k_in, v_in
+
+        out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False)
+
+        out = rearrange(out, "b h n d -> b n (h d)", h=h)
+
+        out = self.to_out[0](out)
+        return out
+
+
+# feedforward
+class GEGLU(nn.Module):
+    r"""
+    A variant of the gated linear unit activation function from https://arxiv.org/abs/2002.05202.
+
+    Parameters:
+        dim_in (`int`): The number of channels in the input.
+        dim_out (`int`): The number of channels in the output.
+    """
+
+    def __init__(self, dim_in: int, dim_out: int):
+        super().__init__()
+        self.proj = nn.Linear(dim_in, dim_out * 2)
+
+    def gelu(self, gate):
+        if gate.device.type != "mps":
+            return F.gelu(gate)
+        # mps: gelu is not implemented for float16
+        return F.gelu(gate.to(dtype=torch.float32)).to(dtype=gate.dtype)
+
+    def forward(self, hidden_states):
+        hidden_states, gate = self.proj(hidden_states).chunk(2, dim=-1)
+        return hidden_states * self.gelu(gate)
+
+
+class FeedForward(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+    ):
+        super().__init__()
+        inner_dim = int(dim * 4)  # mult is always 4
+
+        self.net = nn.ModuleList([])
+        # project in
+        self.net.append(GEGLU(dim, inner_dim))
+        # project dropout
+        self.net.append(nn.Identity())  # nn.Dropout(0)) # dummy for dropout with 0
+        # project out
+        self.net.append(nn.Linear(inner_dim, dim))
+
+    def forward(self, hidden_states):
+        for module in self.net:
+            hidden_states = module(hidden_states)
+        return hidden_states
+
+
+class BasicTransformerBlock(nn.Module):
+    def __init__(
+        self, dim: int, num_attention_heads: int, attention_head_dim: int, cross_attention_dim: int, upcast_attention: bool = False
+    ):
+        super().__init__()
+
+        self.gradient_checkpointing = False
+
+        # 1. Self-Attn
+        self.attn1 = CrossAttention(
+            query_dim=dim,
+            cross_attention_dim=None,
+            heads=num_attention_heads,
+            dim_head=attention_head_dim,
+            upcast_attention=upcast_attention,
+        )
+        self.ff = FeedForward(dim)
+
+        # 2. Cross-Attn
+        self.attn2 = CrossAttention(
+            query_dim=dim,
+            cross_attention_dim=cross_attention_dim,
+            heads=num_attention_heads,
+            dim_head=attention_head_dim,
+            upcast_attention=upcast_attention,
+        )
+
+        self.norm1 = nn.LayerNorm(dim)
+        self.norm2 = nn.LayerNorm(dim)
+
+        # 3. Feed-forward
+        self.norm3 = nn.LayerNorm(dim)
+
+    def set_use_memory_efficient_attention(self, xformers: bool, mem_eff: bool):
+        self.attn1.set_use_memory_efficient_attention(xformers, mem_eff)
+        self.attn2.set_use_memory_efficient_attention(xformers, mem_eff)
+
+    def set_use_sdpa(self, sdpa: bool):
+        self.attn1.set_use_sdpa(sdpa)
+        self.attn2.set_use_sdpa(sdpa)
+
+    def forward_body(self, hidden_states, context=None, timestep=None):
+        # 1. Self-Attention
+        norm_hidden_states = self.norm1(hidden_states)
+
+        hidden_states = self.attn1(norm_hidden_states) + hidden_states
+
+        # 2. Cross-Attention
+        norm_hidden_states = self.norm2(hidden_states)
+        hidden_states = self.attn2(norm_hidden_states, context=context) + hidden_states
+
+        # 3. Feed-forward
+        hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
+
+        return hidden_states
+
+    def forward(self, hidden_states, context=None, timestep=None):
+        if self.training and self.gradient_checkpointing:
+            # print("BasicTransformerBlock: checkpointing")
+
+            def create_custom_forward(func):
+                def custom_forward(*inputs):
+                    return func(*inputs)
+
+                return custom_forward
+
+            output = torch.utils.checkpoint.checkpoint(create_custom_forward(self.forward_body), hidden_states, context, timestep)
+        else:
+            output = self.forward_body(hidden_states, context, timestep)
+
+        return output
+
+
+class Transformer2DModel(nn.Module):
+    def __init__(
+        self,
+        num_attention_heads: int = 16,
+        attention_head_dim: int = 88,
+        in_channels: Optional[int] = None,
+        cross_attention_dim: Optional[int] = None,
+        use_linear_projection: bool = False,
+        upcast_attention: bool = False,
+        num_transformer_layers: int = 1,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.num_attention_heads = num_attention_heads
+        self.attention_head_dim = attention_head_dim
+        inner_dim = num_attention_heads * attention_head_dim
+        self.use_linear_projection = use_linear_projection
+
+        self.norm = torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
+        # self.norm = GroupNorm32(32, in_channels, eps=1e-6, affine=True)
+
+        if use_linear_projection:
+            self.proj_in = nn.Linear(in_channels, inner_dim)
+        else:
+            self.proj_in = nn.Conv2d(in_channels, inner_dim, kernel_size=1, stride=1, padding=0)
+
+        blocks = []
+        for _ in range(num_transformer_layers):
+            blocks.append(
+                BasicTransformerBlock(
+                    inner_dim,
+                    num_attention_heads,
+                    attention_head_dim,
+                    cross_attention_dim=cross_attention_dim,
+                    upcast_attention=upcast_attention,
+                )
+            )
+
+        self.transformer_blocks = nn.ModuleList(blocks)
+
+        if use_linear_projection:
+            self.proj_out = nn.Linear(in_channels, inner_dim)
+        else:
+            self.proj_out = nn.Conv2d(inner_dim, in_channels, kernel_size=1, stride=1, padding=0)
+
+        self.gradient_checkpointing = False
+
+    def set_use_memory_efficient_attention(self, xformers, mem_eff):
+        for transformer in self.transformer_blocks:
+            transformer.set_use_memory_efficient_attention(xformers, mem_eff)
+
+    def set_use_sdpa(self, sdpa):
+        for transformer in self.transformer_blocks:
+            transformer.set_use_sdpa(sdpa)
+
+    def forward(self, hidden_states, encoder_hidden_states=None, timestep=None):
+        # 1. Input
+        batch, _, height, weight = hidden_states.shape
+        residual = hidden_states
+
+        hidden_states = self.norm(hidden_states)
+        if not self.use_linear_projection:
+            hidden_states = self.proj_in(hidden_states)
+            inner_dim = hidden_states.shape[1]
+            hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * weight, inner_dim)
+        else:
+            inner_dim = hidden_states.shape[1]
+            hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * weight, inner_dim)
+            hidden_states = self.proj_in(hidden_states)
+
+        # 2. Blocks
+        for block in self.transformer_blocks:
+            hidden_states = block(hidden_states, context=encoder_hidden_states, timestep=timestep)
+
+        # 3. Output
+        if not self.use_linear_projection:
+            hidden_states = hidden_states.reshape(batch, height, weight, inner_dim).permute(0, 3, 1, 2).contiguous()
+            hidden_states = self.proj_out(hidden_states)
+        else:
+            hidden_states = self.proj_out(hidden_states)
+            hidden_states = hidden_states.reshape(batch, height, weight, inner_dim).permute(0, 3, 1, 2).contiguous()
+
+        output = hidden_states + residual
+
+        return output
+
+    def forward_xxx(self, hidden_states, encoder_hidden_states=None, timestep=None):
+        if self.training and self.gradient_checkpointing:
+            # print("Transformer2DModel: Using gradient checkpointing")
+
+            def create_custom_forward(func):
+                def custom_forward(*inputs):
+                    return func(*inputs)
+
+                return custom_forward
+
+            output = torch.utils.checkpoint.checkpoint(
+                create_custom_forward(self.forward_body), hidden_states, encoder_hidden_states, timestep
+            )
+        else:
+            output = self.forward_body(hidden_states, encoder_hidden_states, timestep)
+
+        return output
+
+
+class Upsample2D(nn.Module):
+    def __init__(self, channels, out_channels):
+        super().__init__()
+        self.channels = channels
+        self.out_channels = out_channels
+        self.conv = nn.Conv2d(self.channels, self.out_channels, 3, padding=1)
+
+        self.gradient_checkpointing = False
+
+    def forward_body(self, hidden_states, output_size=None):
+        assert hidden_states.shape[1] == self.channels
+
+        # Cast to float32 to as 'upsample_nearest2d_out_frame' op does not support bfloat16
+        # TODO(Suraj): Remove this cast once the issue is fixed in PyTorch
+        # https://github.com/pytorch/pytorch/issues/86679
+        dtype = hidden_states.dtype
+        if dtype == torch.bfloat16:
+            hidden_states = hidden_states.to(torch.float32)
+
+        # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/984
+        if hidden_states.shape[0] >= 64:
+            hidden_states = hidden_states.contiguous()
+
+        # if `output_size` is passed we force the interpolation output size and do not make use of `scale_factor=2`
+        if output_size is None:
+            hidden_states = F.interpolate(hidden_states, scale_factor=2.0, mode="nearest")
+        else:
+            hidden_states = F.interpolate(hidden_states, size=output_size, mode="nearest")
+
+        # If the input is bfloat16, we cast back to bfloat16
+        if dtype == torch.bfloat16:
+            hidden_states = hidden_states.to(dtype)
+
+        hidden_states = self.conv(hidden_states)
+
+        return hidden_states
+
+    def forward(self, hidden_states, output_size=None):
+        if self.training and self.gradient_checkpointing:
+            # print("Upsample2D: gradient_checkpointing")
+
+            def create_custom_forward(func):
+                def custom_forward(*inputs):
+                    return func(*inputs)
+
+                return custom_forward
+
+            hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(self.forward_body), hidden_states, output_size)
+        else:
+            hidden_states = self.forward_body(hidden_states, output_size)
+
+        return hidden_states
+
+
+class SdxlUNet2DConditionModel(nn.Module):
+    _supports_gradient_checkpointing = True
+
+    def __init__(
+        self,
+        **kwargs,
+    ):
+        super().__init__()
+
+        self.in_channels = IN_CHANNELS
+        self.out_channels = OUT_CHANNELS
+        self.model_channels = MODEL_CHANNELS
+        self.time_embed_dim = TIME_EMBED_DIM
+        self.adm_in_channels = ADM_IN_CHANNELS
+
+        self.gradient_checkpointing = False
+        # self.sample_size = sample_size
+
+        # time embedding
+        self.time_embed = nn.Sequential(
+            nn.Linear(self.model_channels, self.time_embed_dim),
+            nn.SiLU(),
+            nn.Linear(self.time_embed_dim, self.time_embed_dim),
+        )
+
+        # label embedding
+        self.label_emb = nn.Sequential(
+            nn.Sequential(
+                nn.Linear(self.adm_in_channels, self.time_embed_dim),
+                nn.SiLU(),
+                nn.Linear(self.time_embed_dim, self.time_embed_dim),
+            )
+        )
+
+        # input
+        self.input_blocks = nn.ModuleList(
+            [
+                nn.Sequential(
+                    nn.Conv2d(self.in_channels, self.model_channels, kernel_size=3, padding=(1, 1)),
+                )
+            ]
+        )
+
+        # level 0
+        for i in range(2):
+            layers = [
+                ResnetBlock2D(
+                    in_channels=1 * self.model_channels,
+                    out_channels=1 * self.model_channels,
+                ),
+            ]
+            self.input_blocks.append(nn.ModuleList(layers))
+
+        self.input_blocks.append(
+            nn.Sequential(
+                Downsample2D(
+                    channels=1 * self.model_channels,
+                    out_channels=1 * self.model_channels,
+                ),
+            )
+        )
+
+        # level 1
+        for i in range(2):
+            layers = [
+                ResnetBlock2D(
+                    in_channels=(1 if i == 0 else 2) * self.model_channels,
+                    out_channels=2 * self.model_channels,
+                ),
+                Transformer2DModel(
+                    num_attention_heads=2 * self.model_channels // 64,
+                    attention_head_dim=64,
+                    in_channels=2 * self.model_channels,
+                    num_transformer_layers=2,
+                    use_linear_projection=True,
+                    cross_attention_dim=2048,
+                ),
+            ]
+            self.input_blocks.append(nn.ModuleList(layers))
+
+        self.input_blocks.append(
+            nn.Sequential(
+                Downsample2D(
+                    channels=2 * self.model_channels,
+                    out_channels=2 * self.model_channels,
+                ),
+            )
+        )
+
+        # level 2
+        for i in range(2):
+            layers = [
+                ResnetBlock2D(
+                    in_channels=(2 if i == 0 else 4) * self.model_channels,
+                    out_channels=4 * self.model_channels,
+                ),
+                Transformer2DModel(
+                    num_attention_heads=4 * self.model_channels // 64,
+                    attention_head_dim=64,
+                    in_channels=4 * self.model_channels,
+                    num_transformer_layers=10,
+                    use_linear_projection=True,
+                    cross_attention_dim=2048,
+                ),
+            ]
+            self.input_blocks.append(nn.ModuleList(layers))
+
+        # mid
+        self.middle_block = nn.ModuleList(
+            [
+                ResnetBlock2D(
+                    in_channels=4 * self.model_channels,
+                    out_channels=4 * self.model_channels,
+                ),
+                Transformer2DModel(
+                    num_attention_heads=4 * self.model_channels // 64,
+                    attention_head_dim=64,
+                    in_channels=4 * self.model_channels,
+                    num_transformer_layers=10,
+                    use_linear_projection=True,
+                    cross_attention_dim=2048,
+                ),
+                ResnetBlock2D(
+                    in_channels=4 * self.model_channels,
+                    out_channels=4 * self.model_channels,
+                ),
+            ]
+        )
+
+        # output
+        self.output_blocks = nn.ModuleList([])
+
+        # level 2
+        for i in range(3):
+            layers = [
+                ResnetBlock2D(
+                    in_channels=4 * self.model_channels + (4 if i <= 1 else 2) * self.model_channels,
+                    out_channels=4 * self.model_channels,
+                ),
+                Transformer2DModel(
+                    num_attention_heads=4 * self.model_channels // 64,
+                    attention_head_dim=64,
+                    in_channels=4 * self.model_channels,
+                    num_transformer_layers=10,
+                    use_linear_projection=True,
+                    cross_attention_dim=2048,
+                ),
+            ]
+            if i == 2:
+                layers.append(
+                    Upsample2D(
+                        channels=4 * self.model_channels,
+                        out_channels=4 * self.model_channels,
+                    )
+                )
+
+            self.output_blocks.append(nn.ModuleList(layers))
+
+        # level 1
+        for i in range(3):
+            layers = [
+                ResnetBlock2D(
+                    in_channels=2 * self.model_channels + (4 if i == 0 else (2 if i == 1 else 1)) * self.model_channels,
+                    out_channels=2 * self.model_channels,
+                ),
+                Transformer2DModel(
+                    num_attention_heads=2 * self.model_channels // 64,
+                    attention_head_dim=64,
+                    in_channels=2 * self.model_channels,
+                    num_transformer_layers=2,
+                    use_linear_projection=True,
+                    cross_attention_dim=2048,
+                ),
+            ]
+            if i == 2:
+                layers.append(
+                    Upsample2D(
+                        channels=2 * self.model_channels,
+                        out_channels=2 * self.model_channels,
+                    )
+                )
+
+            self.output_blocks.append(nn.ModuleList(layers))
+
+        # level 0
+        for i in range(3):
+            layers = [
+                ResnetBlock2D(
+                    in_channels=1 * self.model_channels + (2 if i == 0 else 1) * self.model_channels,
+                    out_channels=1 * self.model_channels,
+                ),
+            ]
+
+            self.output_blocks.append(nn.ModuleList(layers))
+
+        # output
+        self.out = nn.ModuleList(
+            [GroupNorm32(32, self.model_channels), nn.SiLU(), nn.Conv2d(self.model_channels, self.out_channels, 3, padding=1)]
+        )
+
+    # region diffusers compatibility
+    def prepare_config(self):
+        self.config = SimpleNamespace()
+
+    @property
+    def dtype(self) -> torch.dtype:
+        # `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype).
+        return get_parameter_dtype(self)
+
+    @property
+    def device(self) -> torch.device:
+        # `torch.device`: The device on which the module is (assuming that all the module parameters are on the same device).
+        return get_parameter_device(self)
+
+    def set_attention_slice(self, slice_size):
+        raise NotImplementedError("Attention slicing is not supported for this model.")
+
+    def is_gradient_checkpointing(self) -> bool:
+        return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules())
+
+    def enable_gradient_checkpointing(self):
+        self.gradient_checkpointing = True
+        self.set_gradient_checkpointing(value=True)
+
+    def disable_gradient_checkpointing(self):
+        self.gradient_checkpointing = False
+        self.set_gradient_checkpointing(value=False)
+
+    def set_use_memory_efficient_attention(self, xformers: bool, mem_eff: bool) -> None:
+        blocks = self.input_blocks + [self.middle_block] + self.output_blocks
+        for block in blocks:
+            for module in block:
+                if hasattr(module, "set_use_memory_efficient_attention"):
+                    # print(module.__class__.__name__)
+                    module.set_use_memory_efficient_attention(xformers, mem_eff)
+
+    def set_use_sdpa(self, sdpa: bool) -> None:
+        blocks = self.input_blocks + [self.middle_block] + self.output_blocks
+        for block in blocks:
+            for module in block:
+                if hasattr(module, "set_use_sdpa"):
+                    module.set_use_sdpa(sdpa)
+
+    def set_gradient_checkpointing(self, value=False):
+        blocks = self.input_blocks + [self.middle_block] + self.output_blocks
+        for block in blocks:
+            for module in block.modules():
+                if hasattr(module, "gradient_checkpointing"):
+                    # print(module.__class__.__name__, module.gradient_checkpointing, "->", value)
+                    module.gradient_checkpointing = value
+
+    # endregion
+
+    def forward(self, x, timesteps=None, context=None, y=None, **kwargs):
+        # broadcast timesteps to batch dimension
+        timesteps = timesteps.expand(x.shape[0])
+
+        hs = []
+        t_emb = get_timestep_embedding(timesteps, self.model_channels)  # , repeat_only=False)
+        t_emb = t_emb.to(x.dtype)
+        emb = self.time_embed(t_emb)
+
+        assert x.shape[0] == y.shape[0], f"batch size mismatch: {x.shape[0]} != {y.shape[0]}"
+        assert x.dtype == y.dtype, f"dtype mismatch: {x.dtype} != {y.dtype}"
+        # assert x.dtype == self.dtype
+        emb = emb + self.label_emb(y)
+
+        def call_module(module, h, emb, context):
+            x = h
+            for layer in module:
+                # print(layer.__class__.__name__, x.dtype, emb.dtype, context.dtype if context is not None else None)
+                if isinstance(layer, ResnetBlock2D):
+                    x = layer(x, emb)
+                elif isinstance(layer, Transformer2DModel):
+                    x = layer(x, context)
+                else:
+                    x = layer(x)
+            return x
+
+        # h = x.type(self.dtype)
+        h = x
+        for module in self.input_blocks:
+            h = call_module(module, h, emb, context)
+            hs.append(h)
+
+        h = call_module(self.middle_block, h, emb, context)
+
+        for module in self.output_blocks:
+            h = torch.cat([h, hs.pop()], dim=1)
+            h = call_module(module, h, emb, context)
+
+        h = h.type(x.dtype)
+        h = call_module(self.out, h, emb, context)
+
+        return h
+
+
+if __name__ == "__main__":
+    import time
+
+    print("create unet")
+    unet = SdxlUNet2DConditionModel()
+
+    unet.to("cuda")
+    unet.set_use_memory_efficient_attention(True, False)
+    unet.set_gradient_checkpointing(True)
+    unet.train()
+
+    # 使用メモリ量確認用の疑似学習ループ
+    print("preparing optimizer")
+
+    # optimizer = torch.optim.SGD(unet.parameters(), lr=1e-3, nesterov=True, momentum=0.9) # not working
+
+    # import bitsandbytes
+    # optimizer = bitsandbytes.adam.Adam8bit(unet.parameters(), lr=1e-3)        # not working
+    # optimizer = bitsandbytes.optim.RMSprop8bit(unet.parameters(), lr=1e-3)  # working at 23.5 GB with torch2
+    # optimizer=bitsandbytes.optim.Adagrad8bit(unet.parameters(), lr=1e-3)  # working at 23.5 GB with torch2
+
+    import transformers
+
+    optimizer = transformers.optimization.Adafactor(unet.parameters(), relative_step=True)  # working at 22.2GB with torch2
+
+    scaler = torch.cuda.amp.GradScaler(enabled=True)
+
+    print("start training")
+    steps = 10
+    batch_size = 1
+
+    for step in range(steps):
+        print(f"step {step}")
+        if step == 1:
+            time_start = time.perf_counter()
+
+        x = torch.randn(batch_size, 4, 128, 128).cuda()  # 1024x1024
+        t = torch.randint(low=0, high=10, size=(batch_size,), device="cuda")
+        ctx = torch.randn(batch_size, 77, 2048).cuda()
+        y = torch.randn(batch_size, ADM_IN_CHANNELS).cuda()
+
+        with torch.cuda.amp.autocast(enabled=True):
+            output = unet(x, t, ctx, y)
+            target = torch.randn_like(output)
+            loss = torch.nn.functional.mse_loss(output, target)
+
+        scaler.scale(loss).backward()
+        scaler.step(optimizer)
+        scaler.update()
+        optimizer.zero_grad(set_to_none=True)
+
+    time_end = time.perf_counter()
+    print(f"elapsed time: {time_end - time_start} [sec] for last {steps - 1} steps")
diff --git a/library/sdxl_train_util.py b/library/sdxl_train_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..0ce0971583ae7f3bcfbd18b23d4daba0518c9bcb
--- /dev/null
+++ b/library/sdxl_train_util.py
@@ -0,0 +1,437 @@
+import argparse
+import gc
+import math
+import os
+from types import SimpleNamespace
+from typing import Any
+import torch
+from tqdm import tqdm
+from transformers import CLIPTokenizer
+import open_clip
+from library import model_util, sdxl_model_util, train_util
+from library.sdxl_lpw_stable_diffusion import SdxlStableDiffusionLongPromptWeightingPipeline
+
+TOKENIZER_PATH = "openai/clip-vit-large-patch14"
+
+DEFAULT_NOISE_OFFSET = 0.0357
+
+
+# TODO: separate checkpoint for each U-Net/Text Encoder/VAE
+def load_target_model(args, accelerator, model_version: str, weight_dtype):
+    # load models for each process
+    for pi in range(accelerator.state.num_processes):
+        if pi == accelerator.state.local_process_index:
+            print(f"loading model for process {accelerator.state.local_process_index}/{accelerator.state.num_processes}")
+
+            (
+                load_stable_diffusion_format,
+                text_encoder1,
+                text_encoder2,
+                vae,
+                unet,
+                logit_scale,
+                ckpt_info,
+            ) = _load_target_model(args, model_version, weight_dtype, accelerator.device if args.lowram else "cpu")
+
+            # work on low-ram device
+            if args.lowram:
+                text_encoder1.to(accelerator.device)
+                text_encoder2.to(accelerator.device)
+                unet.to(accelerator.device)
+                vae.to(accelerator.device)
+
+            gc.collect()
+            torch.cuda.empty_cache()
+        accelerator.wait_for_everyone()
+
+    text_encoder1, text_encoder2, unet = train_util.transform_models_if_DDP([text_encoder1, text_encoder2, unet])
+
+    return load_stable_diffusion_format, text_encoder1, text_encoder2, vae, unet, logit_scale, ckpt_info
+
+
+def _load_target_model(args: argparse.Namespace, model_version: str, weight_dtype, device="cpu"):
+    # only supports StableDiffusion
+    name_or_path = args.pretrained_model_name_or_path
+    name_or_path = os.readlink(name_or_path) if os.path.islink(name_or_path) else name_or_path
+    load_stable_diffusion_format = os.path.isfile(name_or_path)  # determine SD or Diffusers
+    assert (
+        load_stable_diffusion_format
+    ), f"only supports StableDiffusion format for SDXL / SDXLではStableDiffusion形式のみサポートしています: {name_or_path}"
+
+    print(f"load StableDiffusion checkpoint: {name_or_path}")
+    (
+        text_encoder1,
+        text_encoder2,
+        vae,
+        unet,
+        logit_scale,
+        ckpt_info,
+    ) = sdxl_model_util.load_models_from_sdxl_checkpoint(model_version, name_or_path, device)
+
+    # VAEを読み込む
+    if args.vae is not None:
+        vae = model_util.load_vae(args.vae, weight_dtype)
+        print("additional VAE loaded")
+
+    return load_stable_diffusion_format, text_encoder1, text_encoder2, vae, unet, logit_scale, ckpt_info
+
+
+class WrapperTokenizer:
+    # open clipのtokenizerをHuggingFaceのtokenizerと同じ形で使えるようにする
+    # make open clip tokenizer compatible with HuggingFace tokenizer
+    def __init__(self):
+        open_clip_tokenizer = open_clip.tokenizer._tokenizer
+        self.model_max_length = 77
+        self.bos_token_id = open_clip_tokenizer.all_special_ids[0]
+        self.eos_token_id = open_clip_tokenizer.all_special_ids[1]
+        self.pad_token_id = 0  # 結果から推定している assumption from result
+
+    def __call__(self, *args: Any, **kwds: Any) -> Any:
+        return self.tokenize(*args, **kwds)
+
+    def tokenize(self, text, padding=False, truncation=None, max_length=None, return_tensors=None):
+        if padding == "max_length":
+            # for training
+            assert max_length is not None
+            assert truncation == True
+            assert return_tensors == "pt"
+            input_ids = open_clip.tokenize(text, context_length=max_length)
+            return SimpleNamespace(**{"input_ids": input_ids})
+
+        # for weighted prompt
+        assert isinstance(text, str), f"input must be str: {text}"
+
+        input_ids = open_clip.tokenize(text, context_length=self.model_max_length)[0]  # tokenizer returns list
+
+        # find eos
+        eos_index = (input_ids == self.eos_token_id).nonzero().max()
+        input_ids = input_ids[: eos_index + 1]  # include eos
+        return SimpleNamespace(**{"input_ids": input_ids})
+
+    # for Textual Inversion
+    # わりと面倒くさいな……これWeb UIとかでどうするんだろう / this is a bit annoying... how to do this in Web UI?
+
+    def encode(self, text, add_special_tokens=False):
+        assert not add_special_tokens
+        input_ids = open_clip.tokenizer._tokenizer.encode(text)
+        return input_ids
+
+    def add_tokens(self, new_tokens):
+        tokens_to_add = []
+        for token in new_tokens:
+            token = token.lower()
+            if token + "</w>" not in open_clip.tokenizer._tokenizer.encoder:
+                tokens_to_add.append(token)
+
+        # open clipのtokenizerに直接追加する / add tokens to open clip tokenizer
+        for token in tokens_to_add:
+            open_clip.tokenizer._tokenizer.encoder[token + "</w>"] = len(open_clip.tokenizer._tokenizer.encoder)
+            open_clip.tokenizer._tokenizer.decoder[len(open_clip.tokenizer._tokenizer.decoder)] = token + "</w>"
+            open_clip.tokenizer._tokenizer.vocab_size += 1
+
+            # open clipのtokenizerのcacheに直接設定することで、bpeとかいうやつに含まれていなくてもtokenizeできるようにする
+            # めちゃくちゃ乱暴なので、open clipのtokenizerの仕様が変わったら動かなくなる
+            # set cache of open clip tokenizer directly to enable tokenization even if the token is not included in bpe
+            # this is very rough, so it will not work if the specification of open clip tokenizer changes
+            open_clip.tokenizer._tokenizer.cache[token] = token + "</w>"
+
+        return len(tokens_to_add)
+
+    def convert_tokens_to_ids(self, tokens):
+        input_ids = [open_clip.tokenizer._tokenizer.encoder[token + "</w>"] for token in tokens]
+        return input_ids
+
+    def __len__(self):
+        return open_clip.tokenizer._tokenizer.vocab_size
+
+
+def load_tokenizers(args: argparse.Namespace):
+    print("prepare tokenizers")
+    original_path = TOKENIZER_PATH
+
+    tokenizer1: CLIPTokenizer = None
+    if args.tokenizer_cache_dir:
+        local_tokenizer_path = os.path.join(args.tokenizer_cache_dir, original_path.replace("/", "_"))
+        if os.path.exists(local_tokenizer_path):
+            print(f"load tokenizer from cache: {local_tokenizer_path}")
+            tokenizer1 = CLIPTokenizer.from_pretrained(local_tokenizer_path)
+
+    if tokenizer1 is None:
+        tokenizer1 = CLIPTokenizer.from_pretrained(original_path)
+
+    if args.tokenizer_cache_dir and not os.path.exists(local_tokenizer_path):
+        print(f"save Tokenizer to cache: {local_tokenizer_path}")
+        tokenizer1.save_pretrained(local_tokenizer_path)
+
+    if hasattr(args, "max_token_length") and args.max_token_length is not None:
+        print(f"update token length: {args.max_token_length}")
+
+    # tokenizer2 is from open_clip
+    # TODO caching
+    tokenizer2 = WrapperTokenizer()
+
+    return [tokenizer1, tokenizer2]
+
+
+def get_hidden_states(
+    args: argparse.Namespace, input_ids1, input_ids2, tokenizer1, tokenizer2, text_encoder1, text_encoder2, weight_dtype=None
+):
+    # input_ids: b,n,77 -> b*n, 77
+    b_size = input_ids1.size()[0]
+    input_ids1 = input_ids1.reshape((-1, tokenizer1.model_max_length))  # batch_size*n, 77
+    input_ids2 = input_ids2.reshape((-1, tokenizer2.model_max_length))  # batch_size*n, 77
+
+    # text_encoder1
+    enc_out = text_encoder1(input_ids1, output_hidden_states=True, return_dict=True)
+    hidden_states1 = enc_out["hidden_states"][11]
+
+    # text_encoder2
+    enc_out = text_encoder2(input_ids2, output_hidden_states=True, return_dict=True)
+    hidden_states2 = enc_out["hidden_states"][-2]  # penuultimate layer
+    pool2 = enc_out["text_embeds"]
+
+    # b*n, 77, 768 or 1280 -> b, n*77, 768 or 1280
+    n_size = 1 if args.max_token_length is None else args.max_token_length // 75
+    hidden_states1 = hidden_states1.reshape((b_size, -1, hidden_states1.shape[-1]))
+    hidden_states2 = hidden_states2.reshape((b_size, -1, hidden_states2.shape[-1]))
+
+    if args.max_token_length is not None:
+        # bs*3, 77, 768 or 1024
+        # encoder1: <BOS>...<EOS> の三連を <BOS>...<EOS> へ戻す
+        states_list = [hidden_states1[:, 0].unsqueeze(1)]  # <BOS>
+        for i in range(1, args.max_token_length, tokenizer1.model_max_length):
+            states_list.append(hidden_states1[:, i : i + tokenizer1.model_max_length - 2])  # <BOS> の後から <EOS> の前まで
+        states_list.append(hidden_states1[:, -1].unsqueeze(1))  # <EOS>
+        hidden_states1 = torch.cat(states_list, dim=1)
+
+        # v2: <BOS>...<EOS> <PAD> ... の三連を <BOS>...<EOS> <PAD> ... へ戻す　正直この実装でいいのかわからん
+        states_list = [hidden_states2[:, 0].unsqueeze(1)]  # <BOS>
+        for i in range(1, args.max_token_length, tokenizer2.model_max_length):
+            chunk = hidden_states2[:, i : i + tokenizer2.model_max_length - 2]  # <BOS> の後から 最後の前まで
+            # this causes an error:
+            # RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation
+            # if i > 1:
+            #     for j in range(len(chunk)):  # batch_size
+            #         if input_ids2[n_index + j * n_size, 1] == tokenizer2.eos_token_id:  # 空、つまり <BOS> <EOS> <PAD> ...のパターン
+            #             chunk[j, 0] = chunk[j, 1]  # 次の <PAD> の値をコピーする
+            states_list.append(chunk)  # <BOS> の後から <EOS> の前まで
+        states_list.append(hidden_states2[:, -1].unsqueeze(1))  # <EOS> か <PAD> のどちらか
+        hidden_states2 = torch.cat(states_list, dim=1)
+
+        # pool はnの最初のものを使う
+        pool2 = pool2[::n_size]
+
+    if weight_dtype is not None:
+        # this is required for additional network training
+        hidden_states1 = hidden_states1.to(weight_dtype)
+        hidden_states2 = hidden_states2.to(weight_dtype)
+
+    return hidden_states1, hidden_states2, pool2
+
+
+def timestep_embedding(timesteps, dim, max_period=10000):
+    """
+    Create sinusoidal timestep embeddings.
+    :param timesteps: a 1-D Tensor of N indices, one per batch element.
+                      These may be fractional.
+    :param dim: the dimension of the output.
+    :param max_period: controls the minimum frequency of the embeddings.
+    :return: an [N x dim] Tensor of positional embeddings.
+    """
+    half = dim // 2
+    freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to(
+        device=timesteps.device
+    )
+    args = timesteps[:, None].float() * freqs[None]
+    embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+    if dim % 2:
+        embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
+    return embedding
+
+
+def get_timestep_embedding(x, outdim):
+    assert len(x.shape) == 2
+    b, dims = x.shape[0], x.shape[1]
+    x = torch.flatten(x)
+    emb = timestep_embedding(x, outdim)
+    emb = torch.reshape(emb, (b, dims * outdim))
+    return emb
+
+
+def get_size_embeddings(orig_size, crop_size, target_size, device):
+    emb1 = get_timestep_embedding(orig_size, 256)
+    emb2 = get_timestep_embedding(crop_size, 256)
+    emb3 = get_timestep_embedding(target_size, 256)
+    vector = torch.cat([emb1, emb2, emb3], dim=1).to(device)
+    return vector
+
+
+def save_sd_model_on_train_end(
+    args: argparse.Namespace,
+    src_path: str,
+    save_stable_diffusion_format: bool,
+    use_safetensors: bool,
+    save_dtype: torch.dtype,
+    epoch: int,
+    global_step: int,
+    text_encoder1,
+    text_encoder2,
+    unet,
+    vae,
+    logit_scale,
+    ckpt_info,
+):
+    def sd_saver(ckpt_file, epoch_no, global_step):
+        sdxl_model_util.save_stable_diffusion_checkpoint(
+            ckpt_file,
+            text_encoder1,
+            text_encoder2,
+            unet,
+            epoch_no,
+            global_step,
+            ckpt_info,
+            vae,
+            logit_scale,
+            save_dtype,
+        )
+
+    def diffusers_saver(out_dir):
+        raise NotImplementedError("diffusers_saver is not implemented")
+
+    train_util.save_sd_model_on_train_end_common(
+        args, save_stable_diffusion_format, use_safetensors, epoch, global_step, sd_saver, diffusers_saver
+    )
+
+
+# epochとstepの保存、メタデータにepoch/stepが含まれ引数が同じになるため、統合している
+# on_epoch_end: Trueならepoch終了時、Falseならstep経過時
+def save_sd_model_on_epoch_end_or_stepwise(
+    args: argparse.Namespace,
+    on_epoch_end: bool,
+    accelerator,
+    src_path,
+    save_stable_diffusion_format: bool,
+    use_safetensors: bool,
+    save_dtype: torch.dtype,
+    epoch: int,
+    num_train_epochs: int,
+    global_step: int,
+    text_encoder1,
+    text_encoder2,
+    unet,
+    vae,
+    logit_scale,
+    ckpt_info,
+):
+    def sd_saver(ckpt_file, epoch_no, global_step):
+        sdxl_model_util.save_stable_diffusion_checkpoint(
+            ckpt_file,
+            text_encoder1,
+            text_encoder2,
+            unet,
+            epoch_no,
+            global_step,
+            ckpt_info,
+            vae,
+            logit_scale,
+            save_dtype,
+        )
+
+    def diffusers_saver(out_dir):
+        raise NotImplementedError("diffusers_saver is not implemented")
+
+    train_util.save_sd_model_on_epoch_end_or_stepwise_common(
+        args,
+        on_epoch_end,
+        accelerator,
+        save_stable_diffusion_format,
+        use_safetensors,
+        epoch,
+        num_train_epochs,
+        global_step,
+        sd_saver,
+        diffusers_saver,
+    )
+
+
+# TextEncoderの出力をキャッシュする
+# weight_dtypeを指定するとText Encoderそのもの、およひ出力がweight_dtypeになる
+def cache_text_encoder_outputs(args, accelerator, tokenizers, text_encoders, dataset, weight_dtype):
+    print("caching text encoder outputs")
+
+    tokenizer1, tokenizer2 = tokenizers
+    text_encoder1, text_encoder2 = text_encoders
+    text_encoder1.to(accelerator.device)
+    text_encoder2.to(accelerator.device)
+    if weight_dtype is not None:
+        text_encoder1.to(dtype=weight_dtype)
+        text_encoder2.to(dtype=weight_dtype)
+
+    text_encoder1_cache = {}
+    text_encoder2_cache = {}
+    for batch in tqdm(dataset):
+        input_ids1_batch = batch["input_ids"].to(accelerator.device)
+        input_ids2_batch = batch["input_ids2"].to(accelerator.device)
+
+        # split batch to avoid OOM
+        # TODO specify batch size by args
+        for input_id1, input_id2 in zip(input_ids1_batch.split(1), input_ids2_batch.split(1)):
+            # remove input_ids already in cache
+            input_id1_cache_key = tuple(input_id1.flatten().tolist())
+            input_id2_cache_key = tuple(input_id2.flatten().tolist())
+            if input_id1_cache_key in text_encoder1_cache:
+                assert input_id2_cache_key in text_encoder2_cache
+                continue
+
+            with torch.no_grad():
+                encoder_hidden_states1, encoder_hidden_states2, pool2 = get_hidden_states(
+                    args,
+                    input_id1,
+                    input_id2,
+                    tokenizer1,
+                    tokenizer2,
+                    text_encoder1,
+                    text_encoder2,
+                    None if not args.full_fp16 else weight_dtype,
+                )
+            encoder_hidden_states1 = encoder_hidden_states1.detach().to("cpu").squeeze(0)  # n*75+2,768
+            encoder_hidden_states2 = encoder_hidden_states2.detach().to("cpu").squeeze(0)  # n*75+2,1280
+            pool2 = pool2.detach().to("cpu").squeeze(0)  # 1280
+            text_encoder1_cache[input_id1_cache_key] = encoder_hidden_states1
+            text_encoder2_cache[input_id2_cache_key] = (encoder_hidden_states2, pool2)
+    return text_encoder1_cache, text_encoder2_cache
+
+
+def add_sdxl_training_arguments(parser: argparse.ArgumentParser):
+    parser.add_argument(
+        "--cache_text_encoder_outputs", action="store_true", help="cache text encoder outputs / text encoderの出力をキャッシュする"
+    )
+
+
+def verify_sdxl_training_args(args: argparse.Namespace):
+    assert (
+        not args.v2 and not args.v_parameterization
+    ), "v2 or v_parameterization cannot be enabled in SDXL training / SDXL学習ではv2とv_parameterizationを有効にすることはできません"
+    if args.clip_skip is not None:
+        print("clip_skip will be unexpected / SDXL学習ではclip_skipは動作しません")
+
+    if args.multires_noise_iterations:
+        print(
+            f"Warning: SDXL has been trained with noise_offset={DEFAULT_NOISE_OFFSET}, but noise_offset is disabled due to multires_noise_iterations / SDXLはnoise_offset={DEFAULT_NOISE_OFFSET}で学習されていますが、multires_noise_iterationsが有効になっているためnoise_offsetは無効になります"
+        )
+    else:
+        if args.noise_offset is None:
+            args.noise_offset = DEFAULT_NOISE_OFFSET
+        elif args.noise_offset != DEFAULT_NOISE_OFFSET:
+            print(
+                f"Warning: SDXL has been trained with noise_offset={DEFAULT_NOISE_OFFSET} / SDXLはnoise_offset={DEFAULT_NOISE_OFFSET}で学習されています"
+            )
+        print(f"noise_offset is set to {args.noise_offset} / noise_offsetが{args.noise_offset}に設定されました")
+
+    assert (
+        not hasattr(args, "weighted_captions") or not args.weighted_captions
+    ), "weighted_captions cannot be enabled in SDXL training currently / SDXL学習では今のところweighted_captionsを有効にすることはできません"
+
+
+def sample_images(*args, **kwargs):
+    return train_util.sample_images_common(SdxlStableDiffusionLongPromptWeightingPipeline, *args, **kwargs)
diff --git a/library/slicing_vae.py b/library/slicing_vae.py
new file mode 100644
index 0000000000000000000000000000000000000000..490b5a75d850a6f22e9e5a89c005fe57eb8be2ad
--- /dev/null
+++ b/library/slicing_vae.py
@@ -0,0 +1,679 @@
+# Modified from Diffusers to reduce VRAM usage
+
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from dataclasses import dataclass
+from typing import Optional, Tuple, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.modeling_utils import ModelMixin
+from diffusers.utils import BaseOutput
+from diffusers.models.unet_2d_blocks import UNetMidBlock2D, get_down_block, get_up_block, ResnetBlock2D
+from diffusers.models.vae import DecoderOutput, Encoder, AutoencoderKLOutput, DiagonalGaussianDistribution
+
+
+def slice_h(x, num_slices):
+    # slice with pad 1 both sides: to eliminate side effect of padding of conv2d
+    # Conv2dのpaddingの副作用を排除するために、両側にpad 1しながらHをスライスする
+    # NCHWでもNHWCでもどちらでも動く
+    size = (x.shape[2] + num_slices - 1) // num_slices
+    sliced = []
+    for i in range(num_slices):
+        if i == 0:
+            sliced.append(x[:, :, : size + 1, :])
+        else:
+            end = size * (i + 1) + 1
+            if x.shape[2] - end < 3:  # if the last slice is too small, use the rest of the tensor 最後が細すぎるとconv2dできないので全部使う
+                end = x.shape[2]
+            sliced.append(x[:, :, size * i - 1 : end, :])
+            if end >= x.shape[2]:
+                break
+    return sliced
+
+
+def cat_h(sliced):
+    # padding分を除いて結合する
+    cat = []
+    for i, x in enumerate(sliced):
+        if i == 0:
+            cat.append(x[:, :, :-1, :])
+        elif i == len(sliced) - 1:
+            cat.append(x[:, :, 1:, :])
+        else:
+            cat.append(x[:, :, 1:-1, :])
+        del x
+    x = torch.cat(cat, dim=2)
+    return x
+
+
+def resblock_forward(_self, num_slices, input_tensor, temb):
+    assert _self.upsample is None and _self.downsample is None
+    assert _self.norm1.num_groups == _self.norm2.num_groups
+    assert temb is None
+
+    # make sure norms are on cpu
+    org_device = input_tensor.device
+    cpu_device = torch.device("cpu")
+    _self.norm1.to(cpu_device)
+    _self.norm2.to(cpu_device)
+
+    # GroupNormがCPUでfp16で動かない対策
+    org_dtype = input_tensor.dtype
+    if org_dtype == torch.float16:
+        _self.norm1.to(torch.float32)
+        _self.norm2.to(torch.float32)
+
+    # すべてのテンソルをCPUに移動する
+    input_tensor = input_tensor.to(cpu_device)
+    hidden_states = input_tensor
+
+    # どうもこれは結果が異なるようだ……
+    # def sliced_norm1(norm, x):
+    #     num_div = 4 if up_block_idx <= 2 else x.shape[1] // norm.num_groups
+    #     sliced_tensor = torch.chunk(x, num_div, dim=1)
+    #     sliced_weight = torch.chunk(norm.weight, num_div, dim=0)
+    #     sliced_bias = torch.chunk(norm.bias, num_div, dim=0)
+    #     print(sliced_tensor[0].shape, num_div, sliced_weight[0].shape, sliced_bias[0].shape)
+    #     normed_tensor = []
+    #     for i in range(num_div):
+    #         n = torch.group_norm(sliced_tensor[i], norm.num_groups, sliced_weight[i], sliced_bias[i], norm.eps)
+    #         normed_tensor.append(n)
+    #         del n
+    #     x = torch.cat(normed_tensor, dim=1)
+    #     return num_div, x
+
+    # normを分割すると結果が変わるので、ここだけは分割しない。GPUで計算するとVRAMが足りなくなるので、CPUで計算する。幸いCPUでもそこまで遅くない
+    if org_dtype == torch.float16:
+        hidden_states = hidden_states.to(torch.float32)
+    hidden_states = _self.norm1(hidden_states)  # run on cpu
+    if org_dtype == torch.float16:
+        hidden_states = hidden_states.to(torch.float16)
+
+    sliced = slice_h(hidden_states, num_slices)
+    del hidden_states
+
+    for i in range(len(sliced)):
+        x = sliced[i]
+        sliced[i] = None
+
+        # 計算する部分だけGPUに移動する、以下同様
+        x = x.to(org_device)
+        x = _self.nonlinearity(x)
+        x = _self.conv1(x)
+        x = x.to(cpu_device)
+        sliced[i] = x
+        del x
+
+    hidden_states = cat_h(sliced)
+    del sliced
+
+    if org_dtype == torch.float16:
+        hidden_states = hidden_states.to(torch.float32)
+    hidden_states = _self.norm2(hidden_states)  # run on cpu
+    if org_dtype == torch.float16:
+        hidden_states = hidden_states.to(torch.float16)
+
+    sliced = slice_h(hidden_states, num_slices)
+    del hidden_states
+
+    for i in range(len(sliced)):
+        x = sliced[i]
+        sliced[i] = None
+
+        x = x.to(org_device)
+        x = _self.nonlinearity(x)
+        x = _self.dropout(x)
+        x = _self.conv2(x)
+        x = x.to(cpu_device)
+        sliced[i] = x
+        del x
+
+    hidden_states = cat_h(sliced)
+    del sliced
+
+    # make shortcut
+    if _self.conv_shortcut is not None:
+        sliced = list(torch.chunk(input_tensor, num_slices, dim=2))  # no padding in conv_shortcut パディングがないので普通にスライスする
+        del input_tensor
+
+        for i in range(len(sliced)):
+            x = sliced[i]
+            sliced[i] = None
+
+            x = x.to(org_device)
+            x = _self.conv_shortcut(x)
+            x = x.to(cpu_device)
+            sliced[i] = x
+            del x
+
+        input_tensor = torch.cat(sliced, dim=2)
+        del sliced
+
+    output_tensor = (input_tensor + hidden_states) / _self.output_scale_factor
+
+    output_tensor = output_tensor.to(org_device)  # 次のレイヤーがGPUで計算する
+    return output_tensor
+
+
+class SlicingEncoder(nn.Module):
+    def __init__(
+        self,
+        in_channels=3,
+        out_channels=3,
+        down_block_types=("DownEncoderBlock2D",),
+        block_out_channels=(64,),
+        layers_per_block=2,
+        norm_num_groups=32,
+        act_fn="silu",
+        double_z=True,
+        num_slices=2,
+    ):
+        super().__init__()
+        self.layers_per_block = layers_per_block
+
+        self.conv_in = torch.nn.Conv2d(in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1)
+
+        self.mid_block = None
+        self.down_blocks = nn.ModuleList([])
+
+        # down
+        output_channel = block_out_channels[0]
+        for i, down_block_type in enumerate(down_block_types):
+            input_channel = output_channel
+            output_channel = block_out_channels[i]
+            is_final_block = i == len(block_out_channels) - 1
+
+            down_block = get_down_block(
+                down_block_type,
+                num_layers=self.layers_per_block,
+                in_channels=input_channel,
+                out_channels=output_channel,
+                add_downsample=not is_final_block,
+                resnet_eps=1e-6,
+                downsample_padding=0,
+                resnet_act_fn=act_fn,
+                resnet_groups=norm_num_groups,
+                attn_num_head_channels=None,
+                temb_channels=None,
+            )
+            self.down_blocks.append(down_block)
+
+        # mid
+        self.mid_block = UNetMidBlock2D(
+            in_channels=block_out_channels[-1],
+            resnet_eps=1e-6,
+            resnet_act_fn=act_fn,
+            output_scale_factor=1,
+            resnet_time_scale_shift="default",
+            attn_num_head_channels=None,
+            resnet_groups=norm_num_groups,
+            temb_channels=None,
+        )
+        self.mid_block.attentions[0].set_use_memory_efficient_attention_xformers(True)  # とりあえずDiffusersのxformersを使う
+
+        # out
+        self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6)
+        self.conv_act = nn.SiLU()
+
+        conv_out_channels = 2 * out_channels if double_z else out_channels
+        self.conv_out = nn.Conv2d(block_out_channels[-1], conv_out_channels, 3, padding=1)
+
+        # replace forward of ResBlocks
+        def wrapper(func, module, num_slices):
+            def forward(*args, **kwargs):
+                return func(module, num_slices, *args, **kwargs)
+
+            return forward
+
+        self.num_slices = num_slices
+        div = num_slices / (2 ** (len(self.down_blocks) - 1))  # 深い層はそこまで分割しなくていいので適宜減らす
+        # print(f"initial divisor: {div}")
+        if div >= 2:
+            div = int(div)
+            for resnet in self.mid_block.resnets:
+                resnet.forward = wrapper(resblock_forward, resnet, div)
+            # midblock doesn't have downsample
+
+        for i, down_block in enumerate(self.down_blocks[::-1]):
+            if div >= 2:
+                div = int(div)
+                # print(f"down block: {i} divisor: {div}")
+                for resnet in down_block.resnets:
+                    resnet.forward = wrapper(resblock_forward, resnet, div)
+                if down_block.downsamplers is not None:
+                    # print("has downsample")
+                    for downsample in down_block.downsamplers:
+                        downsample.forward = wrapper(self.downsample_forward, downsample, div * 2)
+            div *= 2
+
+    def forward(self, x):
+        sample = x
+        del x
+
+        org_device = sample.device
+        cpu_device = torch.device("cpu")
+
+        # sample = self.conv_in(sample)
+        sample = sample.to(cpu_device)
+        sliced = slice_h(sample, self.num_slices)
+        del sample
+
+        for i in range(len(sliced)):
+            x = sliced[i]
+            sliced[i] = None
+
+            x = x.to(org_device)
+            x = self.conv_in(x)
+            x = x.to(cpu_device)
+            sliced[i] = x
+            del x
+
+        sample = cat_h(sliced)
+        del sliced
+
+        sample = sample.to(org_device)
+
+        # down
+        for down_block in self.down_blocks:
+            sample = down_block(sample)
+
+        # middle
+        sample = self.mid_block(sample)
+
+        # post-process
+        # ここも省メモリ化したいが、恐らくそこまでメモリを食わないので省略
+        sample = self.conv_norm_out(sample)
+        sample = self.conv_act(sample)
+        sample = self.conv_out(sample)
+
+        return sample
+
+    def downsample_forward(self, _self, num_slices, hidden_states):
+        assert hidden_states.shape[1] == _self.channels
+        assert _self.use_conv and _self.padding == 0
+        print("downsample forward", num_slices, hidden_states.shape)
+
+        org_device = hidden_states.device
+        cpu_device = torch.device("cpu")
+
+        hidden_states = hidden_states.to(cpu_device)
+        pad = (0, 1, 0, 1)
+        hidden_states = torch.nn.functional.pad(hidden_states, pad, mode="constant", value=0)
+
+        # slice with even number because of stride 2
+        # strideが2なので偶数でスライスする
+        # slice with pad 1 both sides: to eliminate side effect of padding of conv2d
+        size = (hidden_states.shape[2] + num_slices - 1) // num_slices
+        size = size + 1 if size % 2 == 1 else size
+
+        sliced = []
+        for i in range(num_slices):
+            if i == 0:
+                sliced.append(hidden_states[:, :, : size + 1, :])
+            else:
+                end = size * (i + 1) + 1
+                if hidden_states.shape[2] - end < 4:  # if the last slice is too small, use the rest of the tensor
+                    end = hidden_states.shape[2]
+                sliced.append(hidden_states[:, :, size * i - 1 : end, :])
+                if end >= hidden_states.shape[2]:
+                    break
+        del hidden_states
+
+        for i in range(len(sliced)):
+            x = sliced[i]
+            sliced[i] = None
+
+            x = x.to(org_device)
+            x = _self.conv(x)
+            x = x.to(cpu_device)
+
+            # ここだけ雰囲気が違うのはCopilotのせい
+            if i == 0:
+                hidden_states = x
+            else:
+                hidden_states = torch.cat([hidden_states, x], dim=2)
+
+        hidden_states = hidden_states.to(org_device)
+        # print("downsample forward done", hidden_states.shape)
+        return hidden_states
+
+
+class SlicingDecoder(nn.Module):
+    def __init__(
+        self,
+        in_channels=3,
+        out_channels=3,
+        up_block_types=("UpDecoderBlock2D",),
+        block_out_channels=(64,),
+        layers_per_block=2,
+        norm_num_groups=32,
+        act_fn="silu",
+        num_slices=2,
+    ):
+        super().__init__()
+        self.layers_per_block = layers_per_block
+
+        self.conv_in = nn.Conv2d(in_channels, block_out_channels[-1], kernel_size=3, stride=1, padding=1)
+
+        self.mid_block = None
+        self.up_blocks = nn.ModuleList([])
+
+        # mid
+        self.mid_block = UNetMidBlock2D(
+            in_channels=block_out_channels[-1],
+            resnet_eps=1e-6,
+            resnet_act_fn=act_fn,
+            output_scale_factor=1,
+            resnet_time_scale_shift="default",
+            attn_num_head_channels=None,
+            resnet_groups=norm_num_groups,
+            temb_channels=None,
+        )
+        self.mid_block.attentions[0].set_use_memory_efficient_attention_xformers(True)  # とりあえずDiffusersのxformersを使う
+
+        # up
+        reversed_block_out_channels = list(reversed(block_out_channels))
+        output_channel = reversed_block_out_channels[0]
+        for i, up_block_type in enumerate(up_block_types):
+            prev_output_channel = output_channel
+            output_channel = reversed_block_out_channels[i]
+
+            is_final_block = i == len(block_out_channels) - 1
+
+            up_block = get_up_block(
+                up_block_type,
+                num_layers=self.layers_per_block + 1,
+                in_channels=prev_output_channel,
+                out_channels=output_channel,
+                prev_output_channel=None,
+                add_upsample=not is_final_block,
+                resnet_eps=1e-6,
+                resnet_act_fn=act_fn,
+                resnet_groups=norm_num_groups,
+                attn_num_head_channels=None,
+                temb_channels=None,
+            )
+            self.up_blocks.append(up_block)
+            prev_output_channel = output_channel
+
+        # out
+        self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6)
+        self.conv_act = nn.SiLU()
+        self.conv_out = nn.Conv2d(block_out_channels[0], out_channels, 3, padding=1)
+
+        # replace forward of ResBlocks
+        def wrapper(func, module, num_slices):
+            def forward(*args, **kwargs):
+                return func(module, num_slices, *args, **kwargs)
+
+            return forward
+
+        self.num_slices = num_slices
+        div = num_slices / (2 ** (len(self.up_blocks) - 1))
+        print(f"initial divisor: {div}")
+        if div >= 2:
+            div = int(div)
+            for resnet in self.mid_block.resnets:
+                resnet.forward = wrapper(resblock_forward, resnet, div)
+            # midblock doesn't have upsample
+
+        for i, up_block in enumerate(self.up_blocks):
+            if div >= 2:
+                div = int(div)
+                # print(f"up block: {i} divisor: {div}")
+                for resnet in up_block.resnets:
+                    resnet.forward = wrapper(resblock_forward, resnet, div)
+                if up_block.upsamplers is not None:
+                    # print("has upsample")
+                    for upsample in up_block.upsamplers:
+                        upsample.forward = wrapper(self.upsample_forward, upsample, div * 2)
+            div *= 2
+
+    def forward(self, z):
+        sample = z
+        del z
+        sample = self.conv_in(sample)
+
+        # middle
+        sample = self.mid_block(sample)
+
+        # up
+        for i, up_block in enumerate(self.up_blocks):
+            sample = up_block(sample)
+
+        # post-process
+        sample = self.conv_norm_out(sample)
+        sample = self.conv_act(sample)
+
+        # conv_out with slicing because of VRAM usage
+        # conv_outはとてもVRAM使うのでスライスして対応
+        org_device = sample.device
+        cpu_device = torch.device("cpu")
+        sample = sample.to(cpu_device)
+
+        sliced = slice_h(sample, self.num_slices)
+        del sample
+        for i in range(len(sliced)):
+            x = sliced[i]
+            sliced[i] = None
+
+            x = x.to(org_device)
+            x = self.conv_out(x)
+            x = x.to(cpu_device)
+            sliced[i] = x
+        sample = cat_h(sliced)
+        del sliced
+
+        sample = sample.to(org_device)
+        return sample
+
+    def upsample_forward(self, _self, num_slices, hidden_states, output_size=None):
+        assert hidden_states.shape[1] == _self.channels
+        assert _self.use_conv_transpose == False and _self.use_conv
+
+        org_dtype = hidden_states.dtype
+        org_device = hidden_states.device
+        cpu_device = torch.device("cpu")
+
+        hidden_states = hidden_states.to(cpu_device)
+        sliced = slice_h(hidden_states, num_slices)
+        del hidden_states
+
+        for i in range(len(sliced)):
+            x = sliced[i]
+            sliced[i] = None
+
+            x = x.to(org_device)
+
+            # Cast to float32 to as 'upsample_nearest2d_out_frame' op does not support bfloat16
+            # TODO(Suraj): Remove this cast once the issue is fixed in PyTorch
+            # https://github.com/pytorch/pytorch/issues/86679
+            # PyTorch 2で直らないかね……
+            if org_dtype == torch.bfloat16:
+                x = x.to(torch.float32)
+
+            x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")
+
+            if org_dtype == torch.bfloat16:
+                x = x.to(org_dtype)
+
+            x = _self.conv(x)
+
+            # upsampleされてるのでpadは2になる
+            if i == 0:
+                x = x[:, :, :-2, :]
+            elif i == num_slices - 1:
+                x = x[:, :, 2:, :]
+            else:
+                x = x[:, :, 2:-2, :]
+
+            x = x.to(cpu_device)
+            sliced[i] = x
+            del x
+
+        hidden_states = torch.cat(sliced, dim=2)
+        # print("us hidden_states", hidden_states.shape)
+        del sliced
+
+        hidden_states = hidden_states.to(org_device)
+        return hidden_states
+
+
+class SlicingAutoencoderKL(ModelMixin, ConfigMixin):
+    r"""Variational Autoencoder (VAE) model with KL loss from the paper Auto-Encoding Variational Bayes by Diederik P. Kingma
+    and Max Welling.
+
+    This model inherits from [`ModelMixin`]. Check the superclass documentation for the generic methods the library
+    implements for all the model (such as downloading or saving, etc.)
+
+    Parameters:
+        in_channels (int, *optional*, defaults to 3): Number of channels in the input image.
+        out_channels (int,  *optional*, defaults to 3): Number of channels in the output.
+        down_block_types (`Tuple[str]`, *optional*, defaults to :
+            obj:`("DownEncoderBlock2D",)`): Tuple of downsample block types.
+        up_block_types (`Tuple[str]`, *optional*, defaults to :
+            obj:`("UpDecoderBlock2D",)`): Tuple of upsample block types.
+        block_out_channels (`Tuple[int]`, *optional*, defaults to :
+            obj:`(64,)`): Tuple of block output channels.
+        act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use.
+        latent_channels (`int`, *optional*, defaults to `4`): Number of channels in the latent space.
+        sample_size (`int`, *optional*, defaults to `32`): TODO
+    """
+
+    @register_to_config
+    def __init__(
+        self,
+        in_channels: int = 3,
+        out_channels: int = 3,
+        down_block_types: Tuple[str] = ("DownEncoderBlock2D",),
+        up_block_types: Tuple[str] = ("UpDecoderBlock2D",),
+        block_out_channels: Tuple[int] = (64,),
+        layers_per_block: int = 1,
+        act_fn: str = "silu",
+        latent_channels: int = 4,
+        norm_num_groups: int = 32,
+        sample_size: int = 32,
+        num_slices: int = 16,
+    ):
+        super().__init__()
+
+        # pass init params to Encoder
+        self.encoder = SlicingEncoder(
+            in_channels=in_channels,
+            out_channels=latent_channels,
+            down_block_types=down_block_types,
+            block_out_channels=block_out_channels,
+            layers_per_block=layers_per_block,
+            act_fn=act_fn,
+            norm_num_groups=norm_num_groups,
+            double_z=True,
+            num_slices=num_slices,
+        )
+
+        # pass init params to Decoder
+        self.decoder = SlicingDecoder(
+            in_channels=latent_channels,
+            out_channels=out_channels,
+            up_block_types=up_block_types,
+            block_out_channels=block_out_channels,
+            layers_per_block=layers_per_block,
+            norm_num_groups=norm_num_groups,
+            act_fn=act_fn,
+            num_slices=num_slices,
+        )
+
+        self.quant_conv = torch.nn.Conv2d(2 * latent_channels, 2 * latent_channels, 1)
+        self.post_quant_conv = torch.nn.Conv2d(latent_channels, latent_channels, 1)
+        self.use_slicing = False
+
+    def encode(self, x: torch.FloatTensor, return_dict: bool = True) -> AutoencoderKLOutput:
+        h = self.encoder(x)
+        moments = self.quant_conv(h)
+        posterior = DiagonalGaussianDistribution(moments)
+
+        if not return_dict:
+            return (posterior,)
+
+        return AutoencoderKLOutput(latent_dist=posterior)
+
+    def _decode(self, z: torch.FloatTensor, return_dict: bool = True) -> Union[DecoderOutput, torch.FloatTensor]:
+        z = self.post_quant_conv(z)
+        dec = self.decoder(z)
+
+        if not return_dict:
+            return (dec,)
+
+        return DecoderOutput(sample=dec)
+
+    # これはバッチ方向のスライシング　紛らわしい
+    def enable_slicing(self):
+        r"""
+        Enable sliced VAE decoding.
+
+        When this option is enabled, the VAE will split the input tensor in slices to compute decoding in several
+        steps. This is useful to save some memory and allow larger batch sizes.
+        """
+        self.use_slicing = True
+
+    def disable_slicing(self):
+        r"""
+        Disable sliced VAE decoding. If `enable_slicing` was previously invoked, this method will go back to computing
+        decoding in one step.
+        """
+        self.use_slicing = False
+
+    def decode(self, z: torch.FloatTensor, return_dict: bool = True) -> Union[DecoderOutput, torch.FloatTensor]:
+        if self.use_slicing and z.shape[0] > 1:
+            decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)]
+            decoded = torch.cat(decoded_slices)
+        else:
+            decoded = self._decode(z).sample
+
+        if not return_dict:
+            return (decoded,)
+
+        return DecoderOutput(sample=decoded)
+
+    def forward(
+        self,
+        sample: torch.FloatTensor,
+        sample_posterior: bool = False,
+        return_dict: bool = True,
+        generator: Optional[torch.Generator] = None,
+    ) -> Union[DecoderOutput, torch.FloatTensor]:
+        r"""
+        Args:
+            sample (`torch.FloatTensor`): Input sample.
+            sample_posterior (`bool`, *optional*, defaults to `False`):
+                Whether to sample from the posterior.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`DecoderOutput`] instead of a plain tuple.
+        """
+        x = sample
+        posterior = self.encode(x).latent_dist
+        if sample_posterior:
+            z = posterior.sample(generator=generator)
+        else:
+            z = posterior.mode()
+        dec = self.decode(z).sample
+
+        if not return_dict:
+            return (dec,)
+
+        return DecoderOutput(sample=dec)
diff --git a/library/svd_merge_lora_gui.py b/library/svd_merge_lora_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b5cce95e407546162281ffd1966d5ba4bc3459a
--- /dev/null
+++ b/library/svd_merge_lora_gui.py
@@ -0,0 +1,293 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import (
+    get_saveasfilename_path,
+    get_any_file_path,
+    get_file_path,
+)
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+
+
+def svd_merge_lora(
+    lora_a_model,
+    lora_b_model,
+    lora_c_model,
+    lora_d_model,
+    ratio_a,
+    ratio_b,
+    ratio_c,
+    ratio_d,
+    save_to,
+    precision,
+    save_precision,
+    new_rank,
+    new_conv_rank,
+    device,
+):
+    # Check if the output file already exists
+    if os.path.isfile(save_to):
+        print(f"Output file '{save_to}' already exists. Aborting.")
+        return
+    
+    # Check if the ratio total is equal to one. If not mormalise to 1
+    total_ratio = ratio_a + ratio_b + ratio_c + ratio_d
+    if total_ratio != 1:
+        ratio_a /= total_ratio
+        ratio_b /= total_ratio
+        ratio_c /= total_ratio
+        ratio_d /= total_ratio
+
+    run_cmd = f'{PYTHON} "{os.path.join("networks","svd_merge_lora.py")}"'
+    run_cmd += f' --save_precision {save_precision}'
+    run_cmd += f' --precision {precision}'
+    run_cmd += f' --save_to "{save_to}"'
+
+    run_cmd_models = ' --models'
+    run_cmd_ratios = ' --ratios'
+    # Add non-empty models and their ratios to the command
+    if lora_a_model:
+        if not os.path.isfile(lora_a_model):
+            msgbox('The provided model A is not a file')
+            return
+        run_cmd_models += f' "{lora_a_model}"'
+        run_cmd_ratios += f' {ratio_a}'
+    if lora_b_model:
+        if not os.path.isfile(lora_b_model):
+            msgbox('The provided model B is not a file')
+            return
+        run_cmd_models += f' "{lora_b_model}"'
+        run_cmd_ratios += f' {ratio_b}'
+    if lora_c_model:
+        if not os.path.isfile(lora_c_model):
+            msgbox('The provided model C is not a file')
+            return
+        run_cmd_models += f' "{lora_c_model}"'
+        run_cmd_ratios += f' {ratio_c}'
+    if lora_d_model:
+        if not os.path.isfile(lora_d_model):
+            msgbox('The provided model D is not a file')
+            return
+        run_cmd_models += f' "{lora_d_model}"'
+        run_cmd_ratios += f' {ratio_d}'
+
+    run_cmd += run_cmd_models
+    run_cmd += run_cmd_ratios
+    run_cmd += f' --device {device}'
+    run_cmd += f' --new_rank "{new_rank}"'
+    run_cmd += f' --new_conv_rank "{new_conv_rank}"'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_svd_merge_lora_tab(headless=False):
+    with gr.Tab('Merge LoRA (SVD)'):
+        gr.Markdown('This utility can merge two LoRA networks together into a new LoRA.')
+
+        lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False)
+        lora_ext_name = gr.Textbox(value='LoRA model types', visible=False)
+
+        with gr.Row():
+            lora_a_model = gr.Textbox(
+                label='LoRA model "A"',
+                placeholder='Path to the LoRA A model',
+                interactive=True,
+            )
+            button_lora_a_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_a_model_file.click(
+                get_file_path,
+                inputs=[lora_a_model, lora_ext, lora_ext_name],
+                outputs=lora_a_model,
+                show_progress=False,
+            )
+
+            lora_b_model = gr.Textbox(
+                label='LoRA model "B"',
+                placeholder='Path to the LoRA B model',
+                interactive=True,
+            )
+            button_lora_b_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_b_model_file.click(
+                get_file_path,
+                inputs=[lora_b_model, lora_ext, lora_ext_name],
+                outputs=lora_b_model,
+                show_progress=False,
+            )
+        with gr.Row():
+            ratio_a = gr.Slider(
+                label='Merge ratio model A',
+                minimum=0,
+                maximum=1,
+                step=0.01,
+                value=0.25,
+                interactive=True,
+            )
+            ratio_b = gr.Slider(
+                label='Merge ratio model B',
+                minimum=0,
+                maximum=1,
+                step=0.01,
+                value=0.25,
+                interactive=True,
+            )
+        with gr.Row():
+            lora_c_model = gr.Textbox(
+                label='LoRA model "C"',
+                placeholder='Path to the LoRA C model',
+                interactive=True,
+            )
+            button_lora_c_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_c_model_file.click(
+                get_file_path,
+                inputs=[lora_c_model, lora_ext, lora_ext_name],
+                outputs=lora_c_model,
+                show_progress=False,
+            )
+
+            lora_d_model = gr.Textbox(
+                label='LoRA model "D"',
+                placeholder='Path to the LoRA D model',
+                interactive=True,
+            )
+            button_lora_d_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_d_model_file.click(
+                get_file_path,
+                inputs=[lora_d_model, lora_ext, lora_ext_name],
+                outputs=lora_d_model,
+                show_progress=False,
+            )
+        with gr.Row():
+            ratio_c = gr.Slider(
+                label='Merge ratio model C',
+                minimum=0,
+                maximum=1,
+                step=0.01,
+                value=0.25,
+                interactive=True,
+            )
+            ratio_d = gr.Slider(
+                label='Merge ratio model D',
+                minimum=0,
+                maximum=1,
+                step=0.01,
+                value=0.25,
+                interactive=True,
+            )
+        with gr.Row():
+            new_rank = gr.Slider(
+                label='New Rank',
+                minimum=1,
+                maximum=1024,
+                step=1,
+                value=128,
+                interactive=True,
+            )
+            new_conv_rank = gr.Slider(
+                label='New Conv Rank',
+                minimum=1,
+                maximum=1024,
+                step=1,
+                value=128,
+                interactive=True,
+            )
+
+        with gr.Row():
+            save_to = gr.Textbox(
+                label='Save to',
+                placeholder='path for the new LoRA file to save...',
+                interactive=True,
+            )
+            button_save_to = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_save_to.click(
+                get_saveasfilename_path,
+                inputs=[save_to, lora_ext, lora_ext_name],
+                outputs=save_to,
+                show_progress=False,
+            )
+        with gr.Row():
+            precision = gr.Dropdown(
+                label='Merge precision',
+                choices=['fp16', 'bf16', 'float'],
+                value='float',
+                interactive=True,
+            )
+            save_precision = gr.Dropdown(
+                label='Save precision',
+                choices=['fp16', 'bf16', 'float'],
+                value='float',
+                interactive=True,
+            )
+            device = gr.Dropdown(
+                label='Device',
+                choices=[
+                    'cpu',
+                    'cuda',
+                ],
+                value='cuda',
+                interactive=True,
+            )
+
+        convert_button = gr.Button('Merge model')
+
+        convert_button.click(
+            svd_merge_lora,
+            inputs=[
+                lora_a_model,
+                lora_b_model,
+                lora_c_model,
+                lora_d_model,
+                ratio_a,
+                ratio_b,
+                ratio_c,
+                ratio_d,
+                save_to,
+                precision,
+                save_precision,
+                new_rank,
+                new_conv_rank,
+                device,
+            ],
+            show_progress=False,
+        )
diff --git a/library/tensorboard_gui.py b/library/tensorboard_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a407538024be011c941104c23dfa118bc2a8e72
--- /dev/null
+++ b/library/tensorboard_gui.py
@@ -0,0 +1,58 @@
+import os
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import time
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+tensorboard_proc = None   # I know... bad but heh
+TENSORBOARD = 'tensorboard' if os.name == 'posix' else 'tensorboard.exe'
+
+
+def start_tensorboard(logging_dir):
+    global tensorboard_proc
+
+    if not os.listdir(logging_dir):
+        log.info('Error: log folder is empty')
+        msgbox(msg='Error: log folder is empty')
+        return
+
+    run_cmd = [f'{TENSORBOARD}', '--logdir', f'{logging_dir}', '--host', '0.0.0.0', '--port', '6006']
+
+    log.info(run_cmd)
+    if tensorboard_proc is not None:
+        log.info(
+            'Tensorboard is already running. Terminating existing process before starting new one...'
+        )
+        stop_tensorboard()
+
+    # Start background process
+    log.info('Starting tensorboard...')
+    tensorboard_proc = subprocess.Popen(run_cmd)
+
+    # Wait for some time to allow TensorBoard to start up
+    time.sleep(5)
+
+    # Open the TensorBoard URL in the default browser
+    log.info('Opening tensorboard url in browser...')
+    import webbrowser
+
+    webbrowser.open('http://localhost:6006')
+
+
+def stop_tensorboard():
+    log.info('Stopping tensorboard process...')
+    tensorboard_proc.kill()
+    log.info('...process stopped')
+
+
+def gradio_tensorboard():
+    with gr.Row():
+        button_start_tensorboard = gr.Button('Start tensorboard')
+        button_stop_tensorboard = gr.Button('Stop tensorboard')
+
+    return (button_start_tensorboard, button_stop_tensorboard)
diff --git a/library/train_util.py b/library/train_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..9438a189509b08e5019d7284b7ed70aaef032555
--- /dev/null
+++ b/library/train_util.py
@@ -0,0 +1,4056 @@
+# common functions for training
+
+import argparse
+import ast
+import asyncio
+import importlib
+import json
+import pathlib
+import re
+import shutil
+import time
+from typing import (
+    Dict,
+    List,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
+from accelerate import Accelerator
+import gc
+import glob
+import math
+import os
+import random
+import hashlib
+import subprocess
+from io import BytesIO
+import toml
+
+from tqdm import tqdm
+import torch
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.optim import Optimizer
+from torchvision import transforms
+from transformers import CLIPTokenizer
+import transformers
+from diffusers.optimization import SchedulerType, TYPE_TO_SCHEDULER_FUNCTION
+from diffusers import (
+    StableDiffusionPipeline,
+    DDPMScheduler,
+    EulerAncestralDiscreteScheduler,
+    DPMSolverMultistepScheduler,
+    DPMSolverSinglestepScheduler,
+    LMSDiscreteScheduler,
+    PNDMScheduler,
+    DDIMScheduler,
+    EulerDiscreteScheduler,
+    HeunDiscreteScheduler,
+    KDPM2DiscreteScheduler,
+    KDPM2AncestralDiscreteScheduler,
+)
+from library import custom_train_functions
+from library.original_unet import UNet2DConditionModel
+from huggingface_hub import hf_hub_download
+import albumentations as albu
+import numpy as np
+from PIL import Image
+import cv2
+from einops import rearrange
+from torch import einsum
+import safetensors.torch
+from library.lpw_stable_diffusion import StableDiffusionLongPromptWeightingPipeline
+import library.model_util as model_util
+import library.huggingface_util as huggingface_util
+from library.attention_processors import FlashAttnProcessor
+from library.hypernetwork import replace_attentions_for_hypernetwork
+from library.original_unet import UNet2DConditionModel
+
+# Tokenizer: checkpointから読み込むのではなくあらかじめ提供されているものを使う
+TOKENIZER_PATH = "openai/clip-vit-large-patch14"
+V2_STABLE_DIFFUSION_PATH = "stabilityai/stable-diffusion-2"  # ここからtokenizerだけ使う v2とv2.1はtokenizer仕様は同じ
+
+# checkpointファイル名
+EPOCH_STATE_NAME = "{}-{:06d}-state"
+EPOCH_FILE_NAME = "{}-{:06d}"
+EPOCH_DIFFUSERS_DIR_NAME = "{}-{:06d}"
+LAST_STATE_NAME = "{}-state"
+DEFAULT_EPOCH_NAME = "epoch"
+DEFAULT_LAST_OUTPUT_NAME = "last"
+
+DEFAULT_STEP_NAME = "at"
+STEP_STATE_NAME = "{}-step{:08d}-state"
+STEP_FILE_NAME = "{}-step{:08d}"
+STEP_DIFFUSERS_DIR_NAME = "{}-step{:08d}"
+
+# region dataset
+
+IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".webp", ".bmp", ".PNG", ".JPG", ".JPEG", ".WEBP", ".BMP"]
+
+try:
+    import pillow_avif
+
+    IMAGE_EXTENSIONS.extend([".avif", ".AVIF"])
+except:
+    pass
+
+
+class ImageInfo:
+    def __init__(self, image_key: str, num_repeats: int, caption: str, is_reg: bool, absolute_path: str) -> None:
+        self.image_key: str = image_key
+        self.num_repeats: int = num_repeats
+        self.caption: str = caption
+        self.is_reg: bool = is_reg
+        self.absolute_path: str = absolute_path
+        self.image_size: Tuple[int, int] = None
+        self.resized_size: Tuple[int, int] = None
+        self.bucket_reso: Tuple[int, int] = None
+        self.latents: torch.Tensor = None
+        self.latents_flipped: torch.Tensor = None
+        self.latents_npz: str = None
+        self.latents_npz_flipped: str = None
+        self.latents_original_size: Tuple[int, int] = None  # original image size, not latents size
+        self.latents_crop_left_top: Tuple[int, int] = None  # original image crop left top, not latents crop left top
+        self.cond_img_path: str = None
+
+
+class BucketManager:
+    def __init__(self, no_upscale, max_reso, min_size, max_size, reso_steps) -> None:
+        self.no_upscale = no_upscale
+        if max_reso is None:
+            self.max_reso = None
+            self.max_area = None
+        else:
+            self.max_reso = max_reso
+            self.max_area = max_reso[0] * max_reso[1]
+        self.min_size = min_size
+        self.max_size = max_size
+        self.reso_steps = reso_steps
+
+        self.resos = []
+        self.reso_to_id = {}
+        self.buckets = []  # 前処理時は (image_key, image, original size, crop left/top)、学習時は image_key
+
+    def add_image(self, reso, image_or_info):
+        bucket_id = self.reso_to_id[reso]
+        self.buckets[bucket_id].append(image_or_info)
+
+    def shuffle(self):
+        for bucket in self.buckets:
+            random.shuffle(bucket)
+
+    def sort(self):
+        # 解像度順にソートする（表示時、メタデータ格納時の見栄えをよくするためだけ）。bucketsも入れ替えてreso_to_idも振り直す
+        sorted_resos = self.resos.copy()
+        sorted_resos.sort()
+
+        sorted_buckets = []
+        sorted_reso_to_id = {}
+        for i, reso in enumerate(sorted_resos):
+            bucket_id = self.reso_to_id[reso]
+            sorted_buckets.append(self.buckets[bucket_id])
+            sorted_reso_to_id[reso] = i
+
+        self.resos = sorted_resos
+        self.buckets = sorted_buckets
+        self.reso_to_id = sorted_reso_to_id
+
+    def make_buckets(self):
+        resos = model_util.make_bucket_resolutions(self.max_reso, self.min_size, self.max_size, self.reso_steps)
+        self.set_predefined_resos(resos)
+
+    def set_predefined_resos(self, resos):
+        # 規定サイズから選ぶ場合の解像度、aspect ratioの情報を格納しておく
+        self.predefined_resos = resos.copy()
+        self.predefined_resos_set = set(resos)
+        self.predefined_aspect_ratios = np.array([w / h for w, h in resos])
+
+    def add_if_new_reso(self, reso):
+        if reso not in self.reso_to_id:
+            bucket_id = len(self.resos)
+            self.reso_to_id[reso] = bucket_id
+            self.resos.append(reso)
+            self.buckets.append([])
+            # print(reso, bucket_id, len(self.buckets))
+
+    def round_to_steps(self, x):
+        x = int(x + 0.5)
+        return x - x % self.reso_steps
+
+    def select_bucket(self, image_width, image_height):
+        aspect_ratio = image_width / image_height
+        if not self.no_upscale:
+            # 拡大および縮小を行う
+            # 同じaspect ratioがあるかもしれないので（fine tuningで、no_upscale=Trueで前処理した場合）、解像度が同じものを優先する
+            reso = (image_width, image_height)
+            if reso in self.predefined_resos_set:
+                pass
+            else:
+                ar_errors = self.predefined_aspect_ratios - aspect_ratio
+                predefined_bucket_id = np.abs(ar_errors).argmin()  # 当該解像度以外でaspect ratio errorが最も少ないもの
+                reso = self.predefined_resos[predefined_bucket_id]
+
+            ar_reso = reso[0] / reso[1]
+            if aspect_ratio > ar_reso:  # 横が長い→縦を合わせる
+                scale = reso[1] / image_height
+            else:
+                scale = reso[0] / image_width
+
+            resized_size = (int(image_width * scale + 0.5), int(image_height * scale + 0.5))
+            # print("use predef", image_width, image_height, reso, resized_size)
+        else:
+            # 縮小のみを行う
+            if image_width * image_height > self.max_area:
+                # 画像が大きすぎるのでアスペクト比を保ったまま縮小することを前提にbucketを決める
+                resized_width = math.sqrt(self.max_area * aspect_ratio)
+                resized_height = self.max_area / resized_width
+                assert abs(resized_width / resized_height - aspect_ratio) < 1e-2, "aspect is illegal"
+
+                # リサイズ後の短辺または長辺をreso_steps単位にする：aspect ratioの差が少ないほうを選ぶ
+                # 元のbucketingと同じロジック
+                b_width_rounded = self.round_to_steps(resized_width)
+                b_height_in_wr = self.round_to_steps(b_width_rounded / aspect_ratio)
+                ar_width_rounded = b_width_rounded / b_height_in_wr
+
+                b_height_rounded = self.round_to_steps(resized_height)
+                b_width_in_hr = self.round_to_steps(b_height_rounded * aspect_ratio)
+                ar_height_rounded = b_width_in_hr / b_height_rounded
+
+                # print(b_width_rounded, b_height_in_wr, ar_width_rounded)
+                # print(b_width_in_hr, b_height_rounded, ar_height_rounded)
+
+                if abs(ar_width_rounded - aspect_ratio) < abs(ar_height_rounded - aspect_ratio):
+                    resized_size = (b_width_rounded, int(b_width_rounded / aspect_ratio + 0.5))
+                else:
+                    resized_size = (int(b_height_rounded * aspect_ratio + 0.5), b_height_rounded)
+                # print(resized_size)
+            else:
+                resized_size = (image_width, image_height)  # リサイズは不要
+
+            # 画像のサイズ未満をbucketのサイズとする（paddingせずにcroppingする）
+            bucket_width = resized_size[0] - resized_size[0] % self.reso_steps
+            bucket_height = resized_size[1] - resized_size[1] % self.reso_steps
+            # print("use arbitrary", image_width, image_height, resized_size, bucket_width, bucket_height)
+
+            reso = (bucket_width, bucket_height)
+
+        self.add_if_new_reso(reso)
+
+        ar_error = (reso[0] / reso[1]) - aspect_ratio
+        return reso, resized_size, ar_error
+
+
+class BucketBatchIndex(NamedTuple):
+    bucket_index: int
+    bucket_batch_size: int
+    batch_index: int
+
+
+class AugHelper:
+    def __init__(self):
+        # prepare all possible augmentators
+        self.color_aug_method = albu.OneOf(
+            [
+                albu.HueSaturationValue(8, 0, 0, p=0.5),
+                albu.RandomGamma((95, 105), p=0.5),
+            ],
+            p=0.33,
+        )
+
+        # key: (use_color_aug, use_flip_aug)
+        # self.augmentors = {
+        #     (True, True): albu.Compose(
+        #         [
+        #             color_aug_method,
+        #             flip_aug_method,
+        #         ],
+        #         p=1.0,
+        #     ),
+        #     (True, False): albu.Compose(
+        #         [
+        #             color_aug_method,
+        #         ],
+        #         p=1.0,
+        #     ),
+        #     (False, True): albu.Compose(
+        #         [
+        #             flip_aug_method,
+        #         ],
+        #         p=1.0,
+        #     ),
+        #     (False, False): None,
+        # }
+
+    def get_augmentor(self, use_color_aug: bool) -> Optional[albu.Compose]:
+        return self.color_aug_method if use_color_aug else None
+
+
+class BaseSubset:
+    def __init__(
+        self,
+        image_dir: Optional[str],
+        num_repeats: int,
+        shuffle_caption: bool,
+        keep_tokens: int,
+        color_aug: bool,
+        flip_aug: bool,
+        face_crop_aug_range: Optional[Tuple[float, float]],
+        random_crop: bool,
+        caption_dropout_rate: float,
+        caption_dropout_every_n_epochs: int,
+        caption_tag_dropout_rate: float,
+        token_warmup_min: int,
+        token_warmup_step: Union[float, int],
+    ) -> None:
+        self.image_dir = image_dir
+        self.num_repeats = num_repeats
+        self.shuffle_caption = shuffle_caption
+        self.keep_tokens = keep_tokens
+        self.color_aug = color_aug
+        self.flip_aug = flip_aug
+        self.face_crop_aug_range = face_crop_aug_range
+        self.random_crop = random_crop
+        self.caption_dropout_rate = caption_dropout_rate
+        self.caption_dropout_every_n_epochs = caption_dropout_every_n_epochs
+        self.caption_tag_dropout_rate = caption_tag_dropout_rate
+
+        self.token_warmup_min = token_warmup_min  # step=0におけるタグの数
+        self.token_warmup_step = token_warmup_step  # N（N<1ならN*max_train_steps）ステップ目でタグの数が最大になる
+
+        self.img_count = 0
+
+
+class DreamBoothSubset(BaseSubset):
+    def __init__(
+        self,
+        image_dir: str,
+        is_reg: bool,
+        class_tokens: Optional[str],
+        caption_extension: str,
+        num_repeats,
+        shuffle_caption,
+        keep_tokens,
+        color_aug,
+        flip_aug,
+        face_crop_aug_range,
+        random_crop,
+        caption_dropout_rate,
+        caption_dropout_every_n_epochs,
+        caption_tag_dropout_rate,
+        token_warmup_min,
+        token_warmup_step,
+    ) -> None:
+        assert image_dir is not None, "image_dir must be specified / image_dirは指定が必須です"
+
+        super().__init__(
+            image_dir,
+            num_repeats,
+            shuffle_caption,
+            keep_tokens,
+            color_aug,
+            flip_aug,
+            face_crop_aug_range,
+            random_crop,
+            caption_dropout_rate,
+            caption_dropout_every_n_epochs,
+            caption_tag_dropout_rate,
+            token_warmup_min,
+            token_warmup_step,
+        )
+
+        self.is_reg = is_reg
+        self.class_tokens = class_tokens
+        self.caption_extension = caption_extension
+        if self.caption_extension and not self.caption_extension.startswith("."):
+            self.caption_extension = "." + self.caption_extension
+
+    def __eq__(self, other) -> bool:
+        if not isinstance(other, DreamBoothSubset):
+            return NotImplemented
+        return self.image_dir == other.image_dir
+
+
+class FineTuningSubset(BaseSubset):
+    def __init__(
+        self,
+        image_dir,
+        metadata_file: str,
+        num_repeats,
+        shuffle_caption,
+        keep_tokens,
+        color_aug,
+        flip_aug,
+        face_crop_aug_range,
+        random_crop,
+        caption_dropout_rate,
+        caption_dropout_every_n_epochs,
+        caption_tag_dropout_rate,
+        token_warmup_min,
+        token_warmup_step,
+    ) -> None:
+        assert metadata_file is not None, "metadata_file must be specified / metadata_fileは指定が必須です"
+
+        super().__init__(
+            image_dir,
+            num_repeats,
+            shuffle_caption,
+            keep_tokens,
+            color_aug,
+            flip_aug,
+            face_crop_aug_range,
+            random_crop,
+            caption_dropout_rate,
+            caption_dropout_every_n_epochs,
+            caption_tag_dropout_rate,
+            token_warmup_min,
+            token_warmup_step,
+        )
+
+        self.metadata_file = metadata_file
+
+    def __eq__(self, other) -> bool:
+        if not isinstance(other, FineTuningSubset):
+            return NotImplemented
+        return self.metadata_file == other.metadata_file
+
+
+class ControlNetSubset(BaseSubset):
+    def __init__(
+        self,
+        image_dir: str,
+        conditioning_data_dir: str,
+        caption_extension: str,
+        num_repeats,
+        shuffle_caption,
+        keep_tokens,
+        color_aug,
+        flip_aug,
+        face_crop_aug_range,
+        random_crop,
+        caption_dropout_rate,
+        caption_dropout_every_n_epochs,
+        caption_tag_dropout_rate,
+        token_warmup_min,
+        token_warmup_step,
+    ) -> None:
+        assert image_dir is not None, "image_dir must be specified / image_dirは指定が必須です"
+
+        super().__init__(
+            image_dir,
+            num_repeats,
+            shuffle_caption,
+            keep_tokens,
+            color_aug,
+            flip_aug,
+            face_crop_aug_range,
+            random_crop,
+            caption_dropout_rate,
+            caption_dropout_every_n_epochs,
+            caption_tag_dropout_rate,
+            token_warmup_min,
+            token_warmup_step,
+        )
+
+        self.conditioning_data_dir = conditioning_data_dir
+        self.caption_extension = caption_extension
+        if self.caption_extension and not self.caption_extension.startswith("."):
+            self.caption_extension = "." + self.caption_extension
+
+    def __eq__(self, other) -> bool:
+        if not isinstance(other, ControlNetSubset):
+            return NotImplemented
+        return self.image_dir == other.image_dir and self.conditioning_data_dir == other.conditioning_data_dir
+
+
+class BaseDataset(torch.utils.data.Dataset):
+    def __init__(
+        self,
+        tokenizer: Union[CLIPTokenizer, List[CLIPTokenizer]],
+        max_token_length: int,
+        resolution: Optional[Tuple[int, int]],
+        debug_dataset: bool,
+    ) -> None:
+        super().__init__()
+
+        self.tokenizers = tokenizer if isinstance(tokenizer, list) else [tokenizer]
+
+        self.max_token_length = max_token_length
+        # width/height is used when enable_bucket==False
+        self.width, self.height = (None, None) if resolution is None else resolution
+        self.debug_dataset = debug_dataset
+
+        self.subsets: List[Union[DreamBoothSubset, FineTuningSubset]] = []
+
+        self.token_padding_disabled = False
+        self.tag_frequency = {}
+        self.XTI_layers = None
+        self.token_strings = None
+
+        self.enable_bucket = False
+        self.bucket_manager: BucketManager = None  # not initialized
+        self.min_bucket_reso = None
+        self.max_bucket_reso = None
+        self.bucket_reso_steps = None
+        self.bucket_no_upscale = None
+        self.bucket_info = None  # for metadata
+
+        self.tokenizer_max_length = self.tokenizers[0].model_max_length if max_token_length is None else max_token_length + 2
+
+        self.current_epoch: int = 0  # インスタンスがepochごとに新しく作られるようなので外側から渡さないとダメ
+
+        self.current_step: int = 0
+        self.max_train_steps: int = 0
+        self.seed: int = 0
+
+        # augmentation
+        self.aug_helper = AugHelper()
+
+        self.image_transforms = transforms.Compose(
+            [
+                transforms.ToTensor(),
+                transforms.Normalize([0.5], [0.5]),
+            ]
+        )
+
+        self.image_data: Dict[str, ImageInfo] = {}
+        self.image_to_subset: Dict[str, Union[DreamBoothSubset, FineTuningSubset]] = {}
+
+        self.replacements = {}
+
+    def set_seed(self, seed):
+        self.seed = seed
+
+    def set_current_epoch(self, epoch):
+        if not self.current_epoch == epoch:  # epochが切り替わったらバケツをシャッフルする
+            self.shuffle_buckets()
+        self.current_epoch = epoch
+
+    def set_current_step(self, step):
+        self.current_step = step
+
+    def set_max_train_steps(self, max_train_steps):
+        self.max_train_steps = max_train_steps
+
+    def set_tag_frequency(self, dir_name, captions):
+        frequency_for_dir = self.tag_frequency.get(dir_name, {})
+        self.tag_frequency[dir_name] = frequency_for_dir
+        for caption in captions:
+            for tag in caption.split(","):
+                tag = tag.strip()
+                if tag:
+                    tag = tag.lower()
+                    frequency = frequency_for_dir.get(tag, 0)
+                    frequency_for_dir[tag] = frequency + 1
+
+    def disable_token_padding(self):
+        self.token_padding_disabled = True
+
+    def enable_XTI(self, layers=None, token_strings=None):
+        self.XTI_layers = layers
+        self.token_strings = token_strings
+
+    def add_replacement(self, str_from, str_to):
+        self.replacements[str_from] = str_to
+
+    def process_caption(self, subset: BaseSubset, caption):
+        # dropoutの決定：tag dropがこのメソッド内にあるのでここで行うのが良い
+        is_drop_out = subset.caption_dropout_rate > 0 and random.random() < subset.caption_dropout_rate
+        is_drop_out = (
+            is_drop_out
+            or subset.caption_dropout_every_n_epochs > 0
+            and self.current_epoch % subset.caption_dropout_every_n_epochs == 0
+        )
+
+        if is_drop_out:
+            caption = ""
+        else:
+            if subset.shuffle_caption or subset.token_warmup_step > 0 or subset.caption_tag_dropout_rate > 0:
+                tokens = [t.strip() for t in caption.strip().split(",")]
+                if subset.token_warmup_step < 1:  # 初回に上書きする
+                    subset.token_warmup_step = math.floor(subset.token_warmup_step * self.max_train_steps)
+                if subset.token_warmup_step and self.current_step < subset.token_warmup_step:
+                    tokens_len = (
+                        math.floor((self.current_step) * ((len(tokens) - subset.token_warmup_min) / (subset.token_warmup_step)))
+                        + subset.token_warmup_min
+                    )
+                    tokens = tokens[:tokens_len]
+
+                def dropout_tags(tokens):
+                    if subset.caption_tag_dropout_rate <= 0:
+                        return tokens
+                    l = []
+                    for token in tokens:
+                        if random.random() >= subset.caption_tag_dropout_rate:
+                            l.append(token)
+                    return l
+
+                fixed_tokens = []
+                flex_tokens = tokens[:]
+                if subset.keep_tokens > 0:
+                    fixed_tokens = flex_tokens[: subset.keep_tokens]
+                    flex_tokens = tokens[subset.keep_tokens :]
+
+                if subset.shuffle_caption:
+                    random.shuffle(flex_tokens)
+
+                flex_tokens = dropout_tags(flex_tokens)
+
+                caption = ", ".join(fixed_tokens + flex_tokens)
+
+            # textual inversion対応
+            for str_from, str_to in self.replacements.items():
+                if str_from == "":
+                    # replace all
+                    if type(str_to) == list:
+                        caption = random.choice(str_to)
+                    else:
+                        caption = str_to
+                else:
+                    caption = caption.replace(str_from, str_to)
+
+        return caption
+
+    def get_input_ids(self, caption, tokenizer=None):
+        if tokenizer is None:
+            tokenizer = self.tokenizers[0]
+
+        input_ids = tokenizer(
+            caption, padding="max_length", truncation=True, max_length=self.tokenizer_max_length, return_tensors="pt"
+        ).input_ids
+
+        if self.tokenizer_max_length > tokenizer.model_max_length:
+            input_ids = input_ids.squeeze(0)
+            iids_list = []
+            if tokenizer.pad_token_id == tokenizer.eos_token_id:
+                # v1
+                # 77以上の時は "<BOS> .... <EOS> <EOS> <EOS>" でトータル227とかになっているので、"<BOS>...<EOS>"の三連に変換する
+                # 1111氏のやつは , で区切る、とかしているようだが　とりあえず単純に
+                for i in range(
+                    1, self.tokenizer_max_length - tokenizer.model_max_length + 2, tokenizer.model_max_length - 2
+                ):  # (1, 152, 75)
+                    ids_chunk = (
+                        input_ids[0].unsqueeze(0),
+                        input_ids[i : i + tokenizer.model_max_length - 2],
+                        input_ids[-1].unsqueeze(0),
+                    )
+                    ids_chunk = torch.cat(ids_chunk)
+                    iids_list.append(ids_chunk)
+            else:
+                # v2 or SDXL
+                # 77以上の時は "<BOS> .... <EOS> <PAD> <PAD>..." でトータル227とかになっているので、"<BOS>...<EOS> <PAD> <PAD> ..."の三連に変換する
+                for i in range(1, self.tokenizer_max_length - tokenizer.model_max_length + 2, tokenizer.model_max_length - 2):
+                    ids_chunk = (
+                        input_ids[0].unsqueeze(0),  # BOS
+                        input_ids[i : i + tokenizer.model_max_length - 2],
+                        input_ids[-1].unsqueeze(0),
+                    )  # PAD or EOS
+                    ids_chunk = torch.cat(ids_chunk)
+
+                    # 末尾が <EOS> <PAD> または <PAD> <PAD> の場合は、何もしなくてよい
+                    # 末尾が x <PAD/EOS> の場合は末尾を <EOS> に変える（x <EOS> なら結果的に変化なし）
+                    if ids_chunk[-2] != tokenizer.eos_token_id and ids_chunk[-2] != tokenizer.pad_token_id:
+                        ids_chunk[-1] = tokenizer.eos_token_id
+                    # 先頭が <BOS> <PAD> ... の場合は <BOS> <EOS> <PAD> ... に変える
+                    if ids_chunk[1] == tokenizer.pad_token_id:
+                        ids_chunk[1] = tokenizer.eos_token_id
+
+                    iids_list.append(ids_chunk)
+
+            input_ids = torch.stack(iids_list)  # 3,77
+        return input_ids
+
+    def register_image(self, info: ImageInfo, subset: BaseSubset):
+        self.image_data[info.image_key] = info
+        self.image_to_subset[info.image_key] = subset
+
+    def make_buckets(self):
+        """
+        bucketingを行わない場合も呼び出し必須（ひとつだけbucketを作る）
+        min_size and max_size are ignored when enable_bucket is False
+        """
+        print("loading image sizes.")
+        for info in tqdm(self.image_data.values()):
+            if info.image_size is None:
+                info.image_size = self.get_image_size(info.absolute_path)
+
+        if self.enable_bucket:
+            print("make buckets")
+        else:
+            print("prepare dataset")
+
+        # bucketを作成し、画像をbucketに振り分ける
+        if self.enable_bucket:
+            if self.bucket_manager is None:  # fine tuningの場合でmetadataに定義がある場合は、すでに初期化済み
+                self.bucket_manager = BucketManager(
+                    self.bucket_no_upscale,
+                    (self.width, self.height),
+                    self.min_bucket_reso,
+                    self.max_bucket_reso,
+                    self.bucket_reso_steps,
+                )
+                if not self.bucket_no_upscale:
+                    self.bucket_manager.make_buckets()
+                else:
+                    print(
+                        "min_bucket_reso and max_bucket_reso are ignored if bucket_no_upscale is set, because bucket reso is defined by image size automatically / bucket_no_upscaleが指定された場合は、bucketの解像度は画像サイズから自動計算されるため、min_bucket_resoとmax_bucket_resoは無視されます"
+                    )
+
+            img_ar_errors = []
+            for image_info in self.image_data.values():
+                image_width, image_height = image_info.image_size
+                image_info.bucket_reso, image_info.resized_size, ar_error = self.bucket_manager.select_bucket(
+                    image_width, image_height
+                )
+
+                # print(image_info.image_key, image_info.bucket_reso)
+                img_ar_errors.append(abs(ar_error))
+
+            self.bucket_manager.sort()
+        else:
+            self.bucket_manager = BucketManager(False, (self.width, self.height), None, None, None)
+            self.bucket_manager.set_predefined_resos([(self.width, self.height)])  # ひとつの固定サイズbucketのみ
+            for image_info in self.image_data.values():
+                image_width, image_height = image_info.image_size
+                image_info.bucket_reso, image_info.resized_size, _ = self.bucket_manager.select_bucket(image_width, image_height)
+
+        for image_info in self.image_data.values():
+            for _ in range(image_info.num_repeats):
+                self.bucket_manager.add_image(image_info.bucket_reso, image_info.image_key)
+
+        # bucket情報を表示、格納する
+        if self.enable_bucket:
+            self.bucket_info = {"buckets": {}}
+            print("number of images (including repeats) / 各bucketの画像枚数（繰り返し回数を含む）")
+            for i, (reso, bucket) in enumerate(zip(self.bucket_manager.resos, self.bucket_manager.buckets)):
+                count = len(bucket)
+                if count > 0:
+                    self.bucket_info["buckets"][i] = {"resolution": reso, "count": len(bucket)}
+                    print(f"bucket {i}: resolution {reso}, count: {len(bucket)}")
+
+            img_ar_errors = np.array(img_ar_errors)
+            mean_img_ar_error = np.mean(np.abs(img_ar_errors))
+            self.bucket_info["mean_img_ar_error"] = mean_img_ar_error
+            print(f"mean ar error (without repeats): {mean_img_ar_error}")
+
+        # データ参照用indexを作る。このindexはdatasetのshuffleに用いられる
+        self.buckets_indices: List(BucketBatchIndex) = []
+        for bucket_index, bucket in enumerate(self.bucket_manager.buckets):
+            batch_count = int(math.ceil(len(bucket) / self.batch_size))
+            for batch_index in range(batch_count):
+                self.buckets_indices.append(BucketBatchIndex(bucket_index, self.batch_size, batch_index))
+
+            # ↓以下はbucketごとのbatch件数があまりにも増えて混乱を招くので元に戻す
+            # 　学習時はステップ数がランダムなので、同一画像が同一batch内にあってもそれほど悪影響はないであろう、と考えられる
+            #
+            # # bucketが細分化されることにより、ひとつのbucketに一種類の画像のみというケースが増え、つまりそれは
+            # # ひとつのbatchが同じ画像で占められることになるので、さすがに良くないであろう
+            # # そのためバッチサイズを画像種類までに制限する
+            # # ただそれでも同一画像が同一バッチに含まれる可能性はあるので、繰り返し回数が少ないほうがshuffleの品質は良くなることは間違いない？
+            # # TO DO 正則化画像をepochまたがりで利用する仕組み
+            # num_of_image_types = len(set(bucket))
+            # bucket_batch_size = min(self.batch_size, num_of_image_types)
+            # batch_count = int(math.ceil(len(bucket) / bucket_batch_size))
+            # # print(bucket_index, num_of_image_types, bucket_batch_size, batch_count)
+            # for batch_index in range(batch_count):
+            #   self.buckets_indices.append(BucketBatchIndex(bucket_index, bucket_batch_size, batch_index))
+            # ↑ここまで
+
+        self.shuffle_buckets()
+        self._length = len(self.buckets_indices)
+
+    def shuffle_buckets(self):
+        # set random seed for this epoch
+        random.seed(self.seed + self.current_epoch)
+
+        random.shuffle(self.buckets_indices)
+        self.bucket_manager.shuffle()
+
+    def load_image(self, image_path):
+        image = Image.open(image_path)
+        if not image.mode == "RGB":
+            image = image.convert("RGB")
+        img = np.array(image, np.uint8)
+        return img
+
+    # 画像を読み込む。戻り値はnumpy.ndarray,(original width, original height),(crop left, crop top)
+    def trim_and_resize_if_required(
+        self, subset: BaseSubset, image, reso, resized_size
+    ) -> Tuple[np.ndarray, Tuple[int, int], Tuple[int, int]]:
+        image_height, image_width = image.shape[0:2]
+
+        if image_width != resized_size[0] or image_height != resized_size[1]:
+            # リサイズする
+            image = cv2.resize(image, resized_size, interpolation=cv2.INTER_AREA)  # INTER_AREAでやりたいのでcv2でリサイズ
+
+        image_height, image_width = image.shape[0:2]
+        original_size = (image_width, image_height)
+
+        crop_left_top = (0, 0)
+        if image_width > reso[0]:
+            trim_size = image_width - reso[0]
+            p = trim_size // 2 if not subset.random_crop else random.randint(0, trim_size)
+            # print("w", trim_size, p)
+            image = image[:, p : p + reso[0]]
+            crop_left_top = (p, 0)
+        if image_height > reso[1]:
+            trim_size = image_height - reso[1]
+            p = trim_size // 2 if not subset.random_crop else random.randint(0, trim_size)
+            # print("h", trim_size, p)
+            image = image[p : p + reso[1]]
+            crop_left_top = (crop_left_top[0], p)
+
+        assert (
+            image.shape[0] == reso[1] and image.shape[1] == reso[0]
+        ), f"internal error, illegal trimmed size: {image.shape}, {reso}"
+        return image, original_size, crop_left_top
+
+    def is_latent_cacheable(self):
+        return all([not subset.color_aug and not subset.random_crop for subset in self.subsets])
+
+    def is_text_encoder_output_cacheable(self):
+        return all(
+            [
+                not (
+                    subset.caption_dropout_rate > 0
+                    or subset.shuffle_caption
+                    or subset.token_warmup_step > 0
+                    or subset.caption_tag_dropout_rate > 0
+                )
+                for subset in self.subsets
+            ]
+        )
+
+    def is_disk_cached_latents_is_expected(self, reso, npz_path, flipped_npz_path):
+        expected_latents_size = (reso[1] // 8, reso[0] // 8)  # bucket_resoはWxHなので注意
+
+        for npath in [npz_path, flipped_npz_path]:
+            if npath is None:
+                continue
+            if not os.path.exists(npath):
+                return False
+
+            npz = np.load(npath)
+            if "latents" not in npz or "original_size" not in npz or "crop_left_top" not in npz:  # old ver?
+                return False
+
+            cached_latents = npz["latents"]
+
+            if cached_latents.shape[1:3] != expected_latents_size:
+                return False
+
+        return True
+
+    def cache_latents(self, vae, vae_batch_size=1, cache_to_disk=False, is_main_process=True):
+        # ちょっと速くした
+        print("caching latents.")
+
+        image_infos = list(self.image_data.values())
+
+        # sort by resolution
+        image_infos.sort(key=lambda info: info.bucket_reso[0] * info.bucket_reso[1])
+
+        # split by resolution
+        batches = []
+        batch = []
+        print("checking cache validity...")
+        for info in tqdm(image_infos):
+            subset = self.image_to_subset[info.image_key]
+
+            if info.latents_npz is not None:  # fine tuning dataset
+                continue
+
+            # check disk cache exists and size of latents
+            if cache_to_disk:
+                info.latents_npz = os.path.splitext(info.absolute_path)[0] + ".npz"
+                info.latents_npz_flipped = os.path.splitext(info.absolute_path)[0] + "_flip.npz"
+                if not is_main_process:
+                    continue
+
+                cache_available = self.is_disk_cached_latents_is_expected(
+                    info.bucket_reso, info.latents_npz, info.latents_npz_flipped if subset.flip_aug else None
+                )
+
+                if cache_available:  # do not add to batch
+                    continue
+
+            # if last member of batch has different resolution, flush the batch
+            if len(batch) > 0 and batch[-1].bucket_reso != info.bucket_reso:
+                batches.append(batch)
+                batch = []
+
+            batch.append(info)
+
+            # if number of data in batch is enough, flush the batch
+            if len(batch) >= vae_batch_size:
+                batches.append(batch)
+                batch = []
+
+        if len(batch) > 0:
+            batches.append(batch)
+
+        if cache_to_disk and not is_main_process:  # don't cache latents in non-main process, set to info only
+            return
+
+        # iterate batches
+        for batch in tqdm(batches, smoothing=1, total=len(batches)):
+            images = []
+            for info in batch:
+                image = self.load_image(info.absolute_path)
+                image, original_size, crop_left_top = self.trim_and_resize_if_required(
+                    subset, image, info.bucket_reso, info.resized_size
+                )
+                image = self.image_transforms(image)
+                images.append(image)
+
+                info.latents_original_size = original_size
+                info.latents_crop_left_top = crop_left_top
+
+            img_tensors = torch.stack(images, dim=0)
+            img_tensors = img_tensors.to(device=vae.device, dtype=vae.dtype)
+
+            latents = vae.encode(img_tensors).latent_dist.sample().to("cpu")
+
+            for info, latent in zip(batch, latents):
+                # check NaN
+                if torch.isnan(latents).any():
+                    raise RuntimeError(f"NaN detected in latents: {info.absolute_path}")
+
+                if cache_to_disk:
+                    save_latents_to_disk(info.latents_npz, latent, info.latents_original_size, info.latents_crop_left_top)
+                else:
+                    info.latents = latent
+
+            if subset.flip_aug:
+                img_tensors = torch.flip(img_tensors, dims=[3])
+                latents = vae.encode(img_tensors).latent_dist.sample().to("cpu")
+                for info, latent in zip(batch, latents):
+                    # check NaN
+                    if torch.isnan(latents).any():
+                        raise RuntimeError(f"NaN detected in latents: {info.absolute_path}")
+
+                    if cache_to_disk:
+                        # crop_left_top is reversed when making batch
+                        save_latents_to_disk(
+                            info.latents_npz_flipped, latent, info.latents_original_size, info.latents_crop_left_top
+                        )
+                    else:
+                        info.latents_flipped = latent
+
+    def get_image_size(self, image_path):
+        image = Image.open(image_path)
+        return image.size
+
+    def load_image_with_face_info(self, subset: BaseSubset, image_path: str):
+        img = self.load_image(image_path)
+
+        face_cx = face_cy = face_w = face_h = 0
+        if subset.face_crop_aug_range is not None:
+            tokens = os.path.splitext(os.path.basename(image_path))[0].split("_")
+            if len(tokens) >= 5:
+                face_cx = int(tokens[-4])
+                face_cy = int(tokens[-3])
+                face_w = int(tokens[-2])
+                face_h = int(tokens[-1])
+
+        return img, face_cx, face_cy, face_w, face_h
+
+    # いい感じに切り出す
+    def crop_target(self, subset: BaseSubset, image, face_cx, face_cy, face_w, face_h):
+        height, width = image.shape[0:2]
+        if height == self.height and width == self.width:
+            return image
+
+        # 画像サイズはsizeより大きいのでリサイズする
+        face_size = max(face_w, face_h)
+        size = min(self.height, self.width)  # 短いほう
+        min_scale = max(self.height / height, self.width / width)  # 画像がモデル入力サイズぴったりになる倍率（最小の倍率）
+        min_scale = min(1.0, max(min_scale, size / (face_size * subset.face_crop_aug_range[1])))  # 指定した顔最小サイズ
+        max_scale = min(1.0, max(min_scale, size / (face_size * subset.face_crop_aug_range[0])))  # 指定した顔最大サイズ
+        if min_scale >= max_scale:  # range指定がmin==max
+            scale = min_scale
+        else:
+            scale = random.uniform(min_scale, max_scale)
+
+        nh = int(height * scale + 0.5)
+        nw = int(width * scale + 0.5)
+        assert nh >= self.height and nw >= self.width, f"internal error. small scale {scale}, {width}*{height}"
+        image = cv2.resize(image, (nw, nh), interpolation=cv2.INTER_AREA)
+        face_cx = int(face_cx * scale + 0.5)
+        face_cy = int(face_cy * scale + 0.5)
+        height, width = nh, nw
+
+        # 顔を中心として448*640とかへ切り出す
+        for axis, (target_size, length, face_p) in enumerate(zip((self.height, self.width), (height, width), (face_cy, face_cx))):
+            p1 = face_p - target_size // 2  # 顔を中心に持ってくるための切り出し位置
+
+            if subset.random_crop:
+                # 背景も含めるために顔を中心に置く確率を高めつつずらす
+                range = max(length - face_p, face_p)  # 画像の端から顔中心までの距離の長いほう
+                p1 = p1 + (random.randint(0, range) + random.randint(0, range)) - range  # -range ~ +range までのいい感じの乱数
+            else:
+                # range指定があるときのみ、すこしだけランダムに（わりと適当）
+                if subset.face_crop_aug_range[0] != subset.face_crop_aug_range[1]:
+                    if face_size > size // 10 and face_size >= 40:
+                        p1 = p1 + random.randint(-face_size // 20, +face_size // 20)
+
+            p1 = max(0, min(p1, length - target_size))
+
+            if axis == 0:
+                image = image[p1 : p1 + target_size, :]
+            else:
+                image = image[:, p1 : p1 + target_size]
+
+        return image
+
+    def load_latents_from_npz(self, image_info: ImageInfo, flipped):
+        npz_file = image_info.latents_npz_flipped if flipped else image_info.latents_npz
+        return load_latents_from_disk(npz_file)
+
+    def __len__(self):
+        return self._length
+
+    def __getitem__(self, index):
+        bucket = self.bucket_manager.buckets[self.buckets_indices[index].bucket_index]
+        bucket_batch_size = self.buckets_indices[index].bucket_batch_size
+        image_index = self.buckets_indices[index].batch_index * bucket_batch_size
+
+        loss_weights = []
+        captions = []
+        input_ids_list = []
+        input_ids2_list = []
+        latents_list = []
+        images = []
+        original_sizes_hw = []
+        crop_top_lefts = []
+        target_sizes_hw = []
+        flippeds = []  # 変数名が微妙
+
+        for image_key in bucket[image_index : image_index + bucket_batch_size]:
+            image_info = self.image_data[image_key]
+            subset = self.image_to_subset[image_key]
+            loss_weights.append(self.prior_loss_weight if image_info.is_reg else 1.0)
+
+            flipped = subset.flip_aug and random.random() < 0.5  # not flipped or flipped with 50% chance
+
+            # image/latentsを処理する
+            if image_info.latents is not None:  # cache_latents=Trueの場合
+                original_size = image_info.latents_original_size
+                crop_left_top = image_info.latents_crop_left_top  # calc values later if flipped
+                if not flipped:
+                    latents = image_info.latents
+                else:
+                    latents = image_info.latents_flipped
+
+                image = None
+            elif image_info.latents_npz is not None:  # FineTuningDatasetまたはcache_latents_to_disk=Trueの場合
+                latents, original_size, crop_left_top = self.load_latents_from_npz(image_info, flipped)
+                latents = torch.FloatTensor(latents)
+
+                image = None
+            else:
+                # 画像を読み込み、必要ならcropする
+                img, face_cx, face_cy, face_w, face_h = self.load_image_with_face_info(subset, image_info.absolute_path)
+                im_h, im_w = img.shape[0:2]
+
+                if self.enable_bucket:
+                    img, original_size, crop_left_top = self.trim_and_resize_if_required(
+                        subset, img, image_info.bucket_reso, image_info.resized_size
+                    )
+                else:
+                    if face_cx > 0:  # 顔位置情報あり
+                        img = self.crop_target(subset, img, face_cx, face_cy, face_w, face_h)
+                    elif im_h > self.height or im_w > self.width:
+                        assert (
+                            subset.random_crop
+                        ), f"image too large, but cropping and bucketing are disabled / 画像サイズが大きいのでface_crop_aug_rangeかrandom_crop、またはbucketを有効にしてください: {image_info.absolute_path}"
+                        if im_h > self.height:
+                            p = random.randint(0, im_h - self.height)
+                            img = img[p : p + self.height]
+                        if im_w > self.width:
+                            p = random.randint(0, im_w - self.width)
+                            img = img[:, p : p + self.width]
+
+                    im_h, im_w = img.shape[0:2]
+                    assert (
+                        im_h == self.height and im_w == self.width
+                    ), f"image size is small / 画像サイズが小さいようです: {image_info.absolute_path}"
+
+                    original_size = [im_w, im_h]
+                    crop_left_top = [0, 0]
+
+                # augmentation
+                aug = self.aug_helper.get_augmentor(subset.color_aug)
+                if aug is not None:
+                    img = aug(image=img)["image"]
+
+                if flipped:
+                    img = img[:, ::-1, :].copy()  # copy to avoid negative stride problem
+
+                latents = None
+                image = self.image_transforms(img)  # -1.0~1.0のtorch.Tensorになる
+
+            images.append(image)
+            latents_list.append(latents)
+
+            target_size = (image.shape[2], image.shape[1]) if image is not None else (latents.shape[2] * 8, latents.shape[1] * 8)
+
+            if flipped:
+                crop_left_top = (original_size[0] - crop_left_top[0] - target_size[0], crop_left_top[1])
+
+            original_sizes_hw.append((original_size[1], original_size[0]))
+            crop_top_lefts.append((crop_left_top[1], crop_left_top[0]))
+            target_sizes_hw.append((target_size[1], target_size[0]))
+            flippeds.append(flipped)
+
+            caption = self.process_caption(subset, image_info.caption)
+            if self.XTI_layers:
+                caption_layer = []
+                for layer in self.XTI_layers:
+                    token_strings_from = " ".join(self.token_strings)
+                    token_strings_to = " ".join([f"{x}_{layer}" for x in self.token_strings])
+                    caption_ = caption.replace(token_strings_from, token_strings_to)
+                    caption_layer.append(caption_)
+                captions.append(caption_layer)
+            else:
+                captions.append(caption)
+
+            if not self.token_padding_disabled:  # this option might be omitted in future
+                if self.XTI_layers:
+                    token_caption = self.get_input_ids(caption_layer, self.tokenizers[0])
+                else:
+                    token_caption = self.get_input_ids(caption, self.tokenizers[0])
+                input_ids_list.append(token_caption)
+
+                if len(self.tokenizers) > 1:
+                    if self.XTI_layers:
+                        token_caption2 = self.get_input_ids(caption_layer, self.tokenizers[1])
+                    else:
+                        token_caption2 = self.get_input_ids(caption, self.tokenizers[1])
+                    input_ids2_list.append(token_caption2)
+
+        example = {}
+        example["loss_weights"] = torch.FloatTensor(loss_weights)
+
+        if self.token_padding_disabled:
+            # padding=True means pad in the batch
+            example["input_ids"] = self.tokenizer[0](captions, padding=True, truncation=True, return_tensors="pt").input_ids
+            if len(self.tokenizers) > 1:
+                # following may not work in SDXL, keep the line for future update
+                example["input_ids2"] = self.tokenizer[1](captions, padding=True, truncation=True, return_tensors="pt").input_ids
+        else:
+            example["input_ids"] = torch.stack(input_ids_list)
+            example["input_ids2"] = torch.stack(input_ids2_list) if len(self.tokenizers) > 1 else None
+
+        if images[0] is not None:
+            images = torch.stack(images)
+            images = images.to(memory_format=torch.contiguous_format).float()
+        else:
+            images = None
+        example["images"] = images
+
+        example["latents"] = torch.stack(latents_list) if latents_list[0] is not None else None
+        example["captions"] = captions
+
+        example["original_sizes_hw"] = torch.stack([torch.LongTensor(x) for x in original_sizes_hw])
+        example["crop_top_lefts"] = torch.stack([torch.LongTensor(x) for x in crop_top_lefts])
+        example["target_sizes_hw"] = torch.stack([torch.LongTensor(x) for x in target_sizes_hw])
+        example["flippeds"] = flippeds
+
+        if self.debug_dataset:
+            example["image_keys"] = bucket[image_index : image_index + self.batch_size]
+        return example
+
+
+class DreamBoothDataset(BaseDataset):
+    def __init__(
+        self,
+        subsets: Sequence[DreamBoothSubset],
+        batch_size: int,
+        tokenizer,
+        max_token_length,
+        resolution,
+        enable_bucket: bool,
+        min_bucket_reso: int,
+        max_bucket_reso: int,
+        bucket_reso_steps: int,
+        bucket_no_upscale: bool,
+        prior_loss_weight: float,
+        debug_dataset,
+    ) -> None:
+        super().__init__(tokenizer, max_token_length, resolution, debug_dataset)
+
+        assert resolution is not None, f"resolution is required / resolution（解像度）指定は必須です"
+
+        self.batch_size = batch_size
+        self.size = min(self.width, self.height)  # 短いほう
+        self.prior_loss_weight = prior_loss_weight
+        self.latents_cache = None
+
+        self.enable_bucket = enable_bucket
+        if self.enable_bucket:
+            assert (
+                min(resolution) >= min_bucket_reso
+            ), f"min_bucket_reso must be equal or less than resolution / min_bucket_resoは最小解像度より大きくできません。解像度を大きくするかmin_bucket_resoを小さくしてください"
+            assert (
+                max(resolution) <= max_bucket_reso
+            ), f"max_bucket_reso must be equal or greater than resolution / max_bucket_resoは最大解像度より小さくできません。解像度を小さくするかmin_bucket_resoを大きくしてください"
+            self.min_bucket_reso = min_bucket_reso
+            self.max_bucket_reso = max_bucket_reso
+            self.bucket_reso_steps = bucket_reso_steps
+            self.bucket_no_upscale = bucket_no_upscale
+        else:
+            self.min_bucket_reso = None
+            self.max_bucket_reso = None
+            self.bucket_reso_steps = None  # この情報は使われない
+            self.bucket_no_upscale = False
+
+        def read_caption(img_path, caption_extension):
+            # captionの候補ファイル名を作る
+            base_name = os.path.splitext(img_path)[0]
+            base_name_face_det = base_name
+            tokens = base_name.split("_")
+            if len(tokens) >= 5:
+                base_name_face_det = "_".join(tokens[:-4])
+            cap_paths = [base_name + caption_extension, base_name_face_det + caption_extension]
+
+            caption = None
+            for cap_path in cap_paths:
+                if os.path.isfile(cap_path):
+                    with open(cap_path, "rt", encoding="utf-8") as f:
+                        try:
+                            lines = f.readlines()
+                        except UnicodeDecodeError as e:
+                            print(f"illegal char in file (not UTF-8) / ファイルにUTF-8以外の文字があります: {cap_path}")
+                            raise e
+                        assert len(lines) > 0, f"caption file is empty / キャプションファイルが空です: {cap_path}"
+                        caption = lines[0].strip()
+                    break
+            return caption
+
+        def load_dreambooth_dir(subset: DreamBoothSubset):
+            if not os.path.isdir(subset.image_dir):
+                print(f"not directory: {subset.image_dir}")
+                return [], []
+
+            img_paths = glob_images(subset.image_dir, "*")
+            print(f"found directory {subset.image_dir} contains {len(img_paths)} image files")
+
+            # 画像ファイルごとにプロンプトを読み込み、もしあればそちらを使う
+            captions = []
+            missing_captions = []
+            for img_path in img_paths:
+                cap_for_img = read_caption(img_path, subset.caption_extension)
+                if cap_for_img is None and subset.class_tokens is None:
+                    print(
+                        f"neither caption file nor class tokens are found. use empty caption for {img_path} / キャプションファイルもclass tokenも見つかりませんでした。空のキャプションを使用します: {img_path}"
+                    )
+                    captions.append("")
+                    missing_captions.append(img_path)
+                else:
+                    if cap_for_img is None:
+                        captions.append(subset.class_tokens)
+                        missing_captions.append(img_path)
+                    else:
+                        captions.append(cap_for_img)
+
+            self.set_tag_frequency(os.path.basename(subset.image_dir), captions)  # タグ頻度を記録
+
+            if missing_captions:
+                number_of_missing_captions = len(missing_captions)
+                number_of_missing_captions_to_show = 5
+                remaining_missing_captions = number_of_missing_captions - number_of_missing_captions_to_show
+
+                print(
+                    f"No caption file found for {number_of_missing_captions} images. Training will continue without captions for these images. If class token exists, it will be used. / {number_of_missing_captions}枚の画像にキャプションファイルが見つかりませんでした。これらの画像についてはキャプションなしで学習を続行します。class tokenが存在する場合はそれを使います。"
+                )
+                for i, missing_caption in enumerate(missing_captions):
+                    if i >= number_of_missing_captions_to_show:
+                        print(missing_caption + f"... and {remaining_missing_captions} more")
+                        break
+                    print(missing_caption)
+            return img_paths, captions
+
+        print("prepare images.")
+        num_train_images = 0
+        num_reg_images = 0
+        reg_infos: List[ImageInfo] = []
+        for subset in subsets:
+            if subset.num_repeats < 1:
+                print(
+                    f"ignore subset with image_dir='{subset.image_dir}': num_repeats is less than 1 / num_repeatsが1を下回っているためサブセットを無視します: {subset.num_repeats}"
+                )
+                continue
+
+            if subset in self.subsets:
+                print(
+                    f"ignore duplicated subset with image_dir='{subset.image_dir}': use the first one / 既にサブセットが登録されているため、重複した後発のサブセットを無視します"
+                )
+                continue
+
+            img_paths, captions = load_dreambooth_dir(subset)
+            if len(img_paths) < 1:
+                print(f"ignore subset with image_dir='{subset.image_dir}': no images found / 画像が見つからないためサブセットを無視します")
+                continue
+
+            if subset.is_reg:
+                num_reg_images += subset.num_repeats * len(img_paths)
+            else:
+                num_train_images += subset.num_repeats * len(img_paths)
+
+            for img_path, caption in zip(img_paths, captions):
+                info = ImageInfo(img_path, subset.num_repeats, caption, subset.is_reg, img_path)
+                if subset.is_reg:
+                    reg_infos.append(info)
+                else:
+                    self.register_image(info, subset)
+
+            subset.img_count = len(img_paths)
+            self.subsets.append(subset)
+
+        print(f"{num_train_images} train images with repeating.")
+        self.num_train_images = num_train_images
+
+        print(f"{num_reg_images} reg images.")
+        if num_train_images < num_reg_images:
+            print("some of reg images are not used / 正則化画像の数が多いので、一部使用されない正則化画像があります")
+
+        if num_reg_images == 0:
+            print("no regularization images / 正則化画像が見つかりませんでした")
+        else:
+            # num_repeatsを計算する：どうせ大した数ではないのでループで処理する
+            n = 0
+            first_loop = True
+            while n < num_train_images:
+                for info in reg_infos:
+                    if first_loop:
+                        self.register_image(info, subset)
+                        n += info.num_repeats
+                    else:
+                        info.num_repeats += 1  # rewrite registered info
+                        n += 1
+                    if n >= num_train_images:
+                        break
+                first_loop = False
+
+        self.num_reg_images = num_reg_images
+
+
+class FineTuningDataset(BaseDataset):
+    def __init__(
+        self,
+        subsets: Sequence[FineTuningSubset],
+        batch_size: int,
+        tokenizer,
+        max_token_length,
+        resolution,
+        enable_bucket: bool,
+        min_bucket_reso: int,
+        max_bucket_reso: int,
+        bucket_reso_steps: int,
+        bucket_no_upscale: bool,
+        debug_dataset,
+    ) -> None:
+        super().__init__(tokenizer, max_token_length, resolution, debug_dataset)
+
+        self.batch_size = batch_size
+
+        self.num_train_images = 0
+        self.num_reg_images = 0
+
+        for subset in subsets:
+            if subset.num_repeats < 1:
+                print(
+                    f"ignore subset with metadata_file='{subset.metadata_file}': num_repeats is less than 1 / num_repeatsが1を下回っているためサブセットを無視します: {subset.num_repeats}"
+                )
+                continue
+
+            if subset in self.subsets:
+                print(
+                    f"ignore duplicated subset with metadata_file='{subset.metadata_file}': use the first one / 既にサブセットが登録されているため、重複した後発のサブセットを無視します"
+                )
+                continue
+
+            # メタデータを読み込む
+            if os.path.exists(subset.metadata_file):
+                print(f"loading existing metadata: {subset.metadata_file}")
+                with open(subset.metadata_file, "rt", encoding="utf-8") as f:
+                    metadata = json.load(f)
+            else:
+                raise ValueError(f"no metadata / メタデータファイルがありません: {subset.metadata_file}")
+
+            if len(metadata) < 1:
+                print(f"ignore subset with '{subset.metadata_file}': no image entries found / 画像に関するデータが見つからないためサブセットを無視します")
+                continue
+
+            tags_list = []
+            for image_key, img_md in metadata.items():
+                # path情報を作る
+                abs_path = None
+
+                # まず画像を優先して探す
+                if os.path.exists(image_key):
+                    abs_path = image_key
+                else:
+                    # わりといい加減だがいい方法が思いつかん
+                    paths = glob_images(subset.image_dir, image_key)
+                    if len(paths) > 0:
+                        abs_path = paths[0]
+
+                # なければnpzを探す
+                if abs_path is None:
+                    if os.path.exists(os.path.splitext(image_key)[0] + ".npz"):
+                        abs_path = os.path.splitext(image_key)[0] + ".npz"
+                    else:
+                        npz_path = os.path.join(subset.image_dir, image_key + ".npz")
+                        if os.path.exists(npz_path):
+                            abs_path = npz_path
+
+                assert abs_path is not None, f"no image / 画像がありません: {image_key}"
+
+                caption = img_md.get("caption")
+                tags = img_md.get("tags")
+                if caption is None:
+                    caption = tags
+                elif tags is not None and len(tags) > 0:
+                    caption = caption + ", " + tags
+                    tags_list.append(tags)
+
+                if caption is None:
+                    caption = ""
+
+                image_info = ImageInfo(image_key, subset.num_repeats, caption, False, abs_path)
+                image_info.image_size = img_md.get("train_resolution")
+
+                if not subset.color_aug and not subset.random_crop:
+                    # if npz exists, use them
+                    image_info.latents_npz, image_info.latents_npz_flipped = self.image_key_to_npz_file(subset, image_key)
+
+                self.register_image(image_info, subset)
+
+            self.num_train_images += len(metadata) * subset.num_repeats
+
+            # TODO do not record tag freq when no tag
+            self.set_tag_frequency(os.path.basename(subset.metadata_file), tags_list)
+            subset.img_count = len(metadata)
+            self.subsets.append(subset)
+
+        # check existence of all npz files
+        use_npz_latents = all([not (subset.color_aug or subset.random_crop) for subset in self.subsets])
+        if use_npz_latents:
+            flip_aug_in_subset = False
+            npz_any = False
+            npz_all = True
+
+            for image_info in self.image_data.values():
+                subset = self.image_to_subset[image_info.image_key]
+
+                has_npz = image_info.latents_npz is not None
+                npz_any = npz_any or has_npz
+
+                if subset.flip_aug:
+                    has_npz = has_npz and image_info.latents_npz_flipped is not None
+                    flip_aug_in_subset = True
+                npz_all = npz_all and has_npz
+
+                if npz_any and not npz_all:
+                    break
+
+            if not npz_any:
+                use_npz_latents = False
+                print(f"npz file does not exist. ignore npz files / npzファイルが見つからないためnpzファイルを無視します")
+            elif not npz_all:
+                use_npz_latents = False
+                print(f"some of npz file does not exist. ignore npz files / いくつかのnpzファイルが見つからないためnpzファイルを無視します")
+                if flip_aug_in_subset:
+                    print("maybe no flipped files / 反転されたnpzファイルがないのかもしれません")
+        # else:
+        #   print("npz files are not used with color_aug and/or random_crop / color_augまたはrandom_cropが指定されているためnpzファイルは使用されません")
+
+        # check min/max bucket size
+        sizes = set()
+        resos = set()
+        for image_info in self.image_data.values():
+            if image_info.image_size is None:
+                sizes = None  # not calculated
+                break
+            sizes.add(image_info.image_size[0])
+            sizes.add(image_info.image_size[1])
+            resos.add(tuple(image_info.image_size))
+
+        if sizes is None:
+            if use_npz_latents:
+                use_npz_latents = False
+                print(f"npz files exist, but no bucket info in metadata. ignore npz files / メタデータにbucket情報がないためnpzファイルを無視します")
+
+            assert (
+                resolution is not None
+            ), "if metadata doesn't have bucket info, resolution is required / メタデータにbucket情報がない場合はresolutionを指定してください"
+
+            self.enable_bucket = enable_bucket
+            if self.enable_bucket:
+                self.min_bucket_reso = min_bucket_reso
+                self.max_bucket_reso = max_bucket_reso
+                self.bucket_reso_steps = bucket_reso_steps
+                self.bucket_no_upscale = bucket_no_upscale
+        else:
+            if not enable_bucket:
+                print("metadata has bucket info, enable bucketing / メタデータにbucket情報があるためbucketを有効にします")
+            print("using bucket info in metadata / メタデータ内のbucket情報を使います")
+            self.enable_bucket = True
+
+            assert (
+                not bucket_no_upscale
+            ), "if metadata has bucket info, bucket reso is precalculated, so bucket_no_upscale cannot be used / メタデータ内にbucket情報がある場合はbucketの解像度は計算済みのため、bucket_no_upscaleは使えません"
+
+            # bucket情報を初期化しておく、make_bucketsで再作成しない
+            self.bucket_manager = BucketManager(False, None, None, None, None)
+            self.bucket_manager.set_predefined_resos(resos)
+
+        # npz情報をきれいにしておく
+        if not use_npz_latents:
+            for image_info in self.image_data.values():
+                image_info.latents_npz = image_info.latents_npz_flipped = None
+
+    def image_key_to_npz_file(self, subset: FineTuningSubset, image_key):
+        base_name = os.path.splitext(image_key)[0]
+        npz_file_norm = base_name + ".npz"
+
+        if os.path.exists(npz_file_norm):
+            # image_key is full path
+            npz_file_flip = base_name + "_flip.npz"
+            if not os.path.exists(npz_file_flip):
+                npz_file_flip = None
+            return npz_file_norm, npz_file_flip
+
+        # if not full path, check image_dir. if image_dir is None, return None
+        if subset.image_dir is None:
+            return None, None
+
+        # image_key is relative path
+        npz_file_norm = os.path.join(subset.image_dir, image_key + ".npz")
+        npz_file_flip = os.path.join(subset.image_dir, image_key + "_flip.npz")
+
+        if not os.path.exists(npz_file_norm):
+            npz_file_norm = None
+            npz_file_flip = None
+        elif not os.path.exists(npz_file_flip):
+            npz_file_flip = None
+
+        return npz_file_norm, npz_file_flip
+
+
+class ControlNetDataset(BaseDataset):
+    def __init__(
+        self,
+        subsets: Sequence[ControlNetSubset],
+        batch_size: int,
+        tokenizer,
+        max_token_length,
+        resolution,
+        enable_bucket: bool,
+        min_bucket_reso: int,
+        max_bucket_reso: int,
+        bucket_reso_steps: int,
+        bucket_no_upscale: bool,
+        debug_dataset,
+    ) -> None:
+        super().__init__(tokenizer, max_token_length, resolution, debug_dataset)
+
+        db_subsets = []
+        for subset in subsets:
+            db_subset = DreamBoothSubset(
+                subset.image_dir,
+                False,
+                None,
+                subset.caption_extension,
+                subset.num_repeats,
+                subset.shuffle_caption,
+                subset.keep_tokens,
+                subset.color_aug,
+                subset.flip_aug,
+                subset.face_crop_aug_range,
+                subset.random_crop,
+                subset.caption_dropout_rate,
+                subset.caption_dropout_every_n_epochs,
+                subset.caption_tag_dropout_rate,
+                subset.token_warmup_min,
+                subset.token_warmup_step,
+            )
+            db_subsets.append(db_subset)
+
+        self.dreambooth_dataset_delegate = DreamBoothDataset(
+            db_subsets,
+            batch_size,
+            tokenizer,
+            max_token_length,
+            resolution,
+            enable_bucket,
+            min_bucket_reso,
+            max_bucket_reso,
+            bucket_reso_steps,
+            bucket_no_upscale,
+            1.0,
+            debug_dataset,
+        )
+
+        # config_util等から参照される値をいれておく（若干微妙なのでなんとかしたい）
+        self.image_data = self.dreambooth_dataset_delegate.image_data
+        self.batch_size = batch_size
+        self.num_train_images = self.dreambooth_dataset_delegate.num_train_images
+        self.num_reg_images = self.dreambooth_dataset_delegate.num_reg_images
+
+        # assert all conditioning data exists
+        missing_imgs = []
+        cond_imgs_with_img = set()
+        for image_key, info in self.dreambooth_dataset_delegate.image_data.items():
+            db_subset = self.dreambooth_dataset_delegate.image_to_subset[image_key]
+            subset = None
+            for s in subsets:
+                if s.image_dir == db_subset.image_dir:
+                    subset = s
+                    break
+            assert subset is not None, "internal error: subset not found"
+
+            if not os.path.isdir(subset.conditioning_data_dir):
+                print(f"not directory: {subset.conditioning_data_dir}")
+                continue
+
+            img_basename = os.path.basename(info.absolute_path)
+            ctrl_img_path = os.path.join(subset.conditioning_data_dir, img_basename)
+            if not os.path.exists(ctrl_img_path):
+                missing_imgs.append(img_basename)
+
+            info.cond_img_path = ctrl_img_path
+            cond_imgs_with_img.add(ctrl_img_path)
+
+        extra_imgs = []
+        for subset in subsets:
+            conditioning_img_paths = glob_images(subset.conditioning_data_dir, "*")
+            extra_imgs.extend(
+                [cond_img_path for cond_img_path in conditioning_img_paths if cond_img_path not in cond_imgs_with_img]
+            )
+
+        assert len(missing_imgs) == 0, f"missing conditioning data for {len(missing_imgs)} images: {missing_imgs}"
+        assert len(extra_imgs) == 0, f"extra conditioning data for {len(extra_imgs)} images: {extra_imgs}"
+
+        self.conditioning_image_transforms = transforms.Compose(
+            [
+                transforms.ToTensor(),
+            ]
+        )
+
+    def make_buckets(self):
+        self.dreambooth_dataset_delegate.make_buckets()
+        self.bucket_manager = self.dreambooth_dataset_delegate.bucket_manager
+        self.buckets_indices = self.dreambooth_dataset_delegate.buckets_indices
+
+    def __len__(self):
+        return self.dreambooth_dataset_delegate.__len__()
+
+    def __getitem__(self, index):
+        example = self.dreambooth_dataset_delegate[index]
+
+        bucket = self.dreambooth_dataset_delegate.bucket_manager.buckets[
+            self.dreambooth_dataset_delegate.buckets_indices[index].bucket_index
+        ]
+        bucket_batch_size = self.dreambooth_dataset_delegate.buckets_indices[index].bucket_batch_size
+        image_index = self.dreambooth_dataset_delegate.buckets_indices[index].batch_index * bucket_batch_size
+
+        conditioning_images = []
+
+        for i, image_key in enumerate(bucket[image_index : image_index + bucket_batch_size]):
+            image_info = self.dreambooth_dataset_delegate.image_data[image_key]
+
+            target_size_hw = example["target_sizes_hw"][i]
+            original_size_hw = example["original_sizes_hw"][i]
+            crop_top_left = example["crop_top_lefts"][i]
+            flipped = example["flippeds"][i]
+            cond_img = self.load_image(image_info.cond_img_path)
+
+            if self.dreambooth_dataset_delegate.enable_bucket:
+                cond_img = cv2.resize(cond_img, image_info.resized_size, interpolation=cv2.INTER_AREA)  # INTER_AREAでやりたいのでcv2でリサイズ
+                assert (
+                    cond_img.shape[0] == original_size_hw[0] and cond_img.shape[1] == original_size_hw[1]
+                ), f"size of conditioning image is not match / 画像サイズが合いません: {image_info.absolute_path}"
+                ct, cl = crop_top_left
+                h, w = target_size_hw
+                cond_img = cond_img[ct : ct + h, cl : cl + w]
+            else:
+                assert (
+                    cond_img.shape[0] == self.height and cond_img.shape[1] == self.width
+                ), f"image size is small / 画像サイズが小さいようです: {image_info.absolute_path}"
+
+            if flipped:
+                cond_img = cond_img[:, ::-1, :].copy()  # copy to avoid negative stride
+
+            cond_img = self.conditioning_image_transforms(cond_img)
+            conditioning_images.append(cond_img)
+
+        example["conditioning_images"] = torch.stack(conditioning_images).to(memory_format=torch.contiguous_format).float()
+
+        return example
+
+
+# behave as Dataset mock
+class DatasetGroup(torch.utils.data.ConcatDataset):
+    def __init__(self, datasets: Sequence[Union[DreamBoothDataset, FineTuningDataset]]):
+        self.datasets: List[Union[DreamBoothDataset, FineTuningDataset]]
+
+        super().__init__(datasets)
+
+        self.image_data = {}
+        self.num_train_images = 0
+        self.num_reg_images = 0
+
+        # simply concat together
+        # TODO: handling image_data key duplication among dataset
+        #   In practical, this is not the big issue because image_data is accessed from outside of dataset only for debug_dataset.
+        for dataset in datasets:
+            self.image_data.update(dataset.image_data)
+            self.num_train_images += dataset.num_train_images
+            self.num_reg_images += dataset.num_reg_images
+
+    def add_replacement(self, str_from, str_to):
+        for dataset in self.datasets:
+            dataset.add_replacement(str_from, str_to)
+
+    # def make_buckets(self):
+    #   for dataset in self.datasets:
+    #     dataset.make_buckets()
+
+    def enable_XTI(self, *args, **kwargs):
+        for dataset in self.datasets:
+            dataset.enable_XTI(*args, **kwargs)
+
+    def cache_latents(self, vae, vae_batch_size=1, cache_to_disk=False, is_main_process=True):
+        for i, dataset in enumerate(self.datasets):
+            print(f"[Dataset {i}]")
+            dataset.cache_latents(vae, vae_batch_size, cache_to_disk, is_main_process)
+
+    def is_latent_cacheable(self) -> bool:
+        return all([dataset.is_latent_cacheable() for dataset in self.datasets])
+
+    def is_text_encoder_output_cacheable(self) -> bool:
+        return all([dataset.is_text_encoder_output_cacheable() for dataset in self.datasets])
+
+    def set_current_epoch(self, epoch):
+        for dataset in self.datasets:
+            dataset.set_current_epoch(epoch)
+
+    def set_current_step(self, step):
+        for dataset in self.datasets:
+            dataset.set_current_step(step)
+
+    def set_max_train_steps(self, max_train_steps):
+        for dataset in self.datasets:
+            dataset.set_max_train_steps(max_train_steps)
+
+    def disable_token_padding(self):
+        for dataset in self.datasets:
+            dataset.disable_token_padding()
+
+
+# 戻り値は、latents_tensor, (original_size width, original_size height), (crop left, crop top)
+def load_latents_from_disk(npz_path) -> Tuple[Optional[torch.Tensor], Optional[List[int]], Optional[List[int]]]:
+    if npz_path is None:  # flipped doesn't exist
+        return None, None, None
+
+    npz = np.load(npz_path)
+    if "latents" not in npz:
+        print(f"error: npz is old format. please re-generate {npz_path}")
+        return None, None, None
+
+    latents = npz["latents"]
+    original_size = npz["original_size"].tolist()
+    crop_left_top = npz["crop_left_top"].tolist()
+    return latents, original_size, crop_left_top
+
+
+def save_latents_to_disk(npz_path, latents_tensor, original_size, crop_left_top):
+    np.savez(
+        npz_path,
+        latents=latents_tensor.float().cpu().numpy(),
+        original_size=np.array(original_size),
+        crop_left_top=np.array(crop_left_top),
+    )
+
+
+def debug_dataset(train_dataset, show_input_ids=False):
+    print(f"Total dataset length (steps) / データセットの長さ（ステップ数）: {len(train_dataset)}")
+    print("`S` for next step, `E` for next epoch no. , Escape for exit. / Sキーで次のステップ、Eキーで次のエポック、Escキーで中断、終了します")
+
+    epoch = 1
+    while True:
+        print(f"\nepoch: {epoch}")
+
+        steps = (epoch - 1) * len(train_dataset) + 1
+        indices = list(range(len(train_dataset)))
+        random.shuffle(indices)
+
+        k = 0
+        for i, idx in enumerate(indices):
+            train_dataset.set_current_epoch(epoch)
+            train_dataset.set_current_step(steps)
+            print(f"steps: {steps} ({i + 1}/{len(train_dataset)})")
+
+            example = train_dataset[idx]
+            if example["latents"] is not None:
+                print(f"sample has latents from npz file: {example['latents'].size()}")
+            for j, (ik, cap, lw, iid, orgsz, crptl, trgsz, flpdz) in enumerate(
+                zip(
+                    example["image_keys"],
+                    example["captions"],
+                    example["loss_weights"],
+                    example["input_ids"],
+                    example["original_sizes_hw"],
+                    example["crop_top_lefts"],
+                    example["target_sizes_hw"],
+                    example["flippeds"],
+                )
+            ):
+                print(
+                    f'{ik}, size: {train_dataset.image_data[ik].image_size}, loss weight: {lw}, caption: "{cap}", original size: {orgsz}, crop left top: {crptl}, target size: {trgsz}, flipped: {flpdz}'
+                )
+
+                if show_input_ids:
+                    print(f"input ids: {iid}")
+                    if "input_ids2" in example:
+                        print(f"input ids2: {example['input_ids2'][j]}")
+                if example["images"] is not None:
+                    im = example["images"][j]
+                    print(f"image size: {im.size()}")
+                    im = ((im.numpy() + 1.0) * 127.5).astype(np.uint8)
+                    im = np.transpose(im, (1, 2, 0))  # c,H,W -> H,W,c
+                    im = im[:, :, ::-1]  # RGB -> BGR (OpenCV)
+
+                    if "conditioning_images" in example:
+                        cond_img = example["conditioning_images"][j]
+                        print(f"conditioning image size: {cond_img.size()}")
+                        cond_img = (cond_img.numpy() * 255.0).astype(np.uint8)
+                        cond_img = np.transpose(cond_img, (1, 2, 0))
+                        cond_img = cond_img[:, :, ::-1]
+                        if os.name == "nt":
+                            cv2.imshow("cond_img", cond_img)
+
+                    if os.name == "nt":  # only windows
+                        cv2.imshow("img", im)
+                        k = cv2.waitKey()
+                        cv2.destroyAllWindows()
+                    if k == 27 or k == ord("s") or k == ord("e"):
+                        break
+            steps += 1
+
+            if k == ord("e"):
+                break
+            if k == 27 or (example["images"] is None and i >= 8):
+                k = 27
+                break
+        if k == 27:
+            break
+
+        epoch += 1
+
+
+def glob_images(directory, base="*"):
+    img_paths = []
+    for ext in IMAGE_EXTENSIONS:
+        if base == "*":
+            img_paths.extend(glob.glob(os.path.join(glob.escape(directory), base + ext)))
+        else:
+            img_paths.extend(glob.glob(glob.escape(os.path.join(directory, base + ext))))
+    img_paths = list(set(img_paths))  # 重複を排除
+    img_paths.sort()
+    return img_paths
+
+
+def glob_images_pathlib(dir_path, recursive):
+    image_paths = []
+    if recursive:
+        for ext in IMAGE_EXTENSIONS:
+            image_paths += list(dir_path.rglob("*" + ext))
+    else:
+        for ext in IMAGE_EXTENSIONS:
+            image_paths += list(dir_path.glob("*" + ext))
+    image_paths = list(set(image_paths))  # 重複を排除
+    image_paths.sort()
+    return image_paths
+
+
+class MinimalDataset(BaseDataset):
+    def __init__(self, tokenizer, max_token_length, resolution, debug_dataset=False):
+        super().__init__(tokenizer, max_token_length, resolution, debug_dataset)
+
+        self.num_train_images = 0  # update in subclass
+        self.num_reg_images = 0  # update in subclass
+        self.datasets = [self]
+        self.batch_size = 1  # update in subclass
+
+        self.subsets = [self]
+        self.num_repeats = 1  # update in subclass if needed
+        self.img_count = 1  # update in subclass if needed
+        self.bucket_info = {}
+        self.is_reg = False
+        self.image_dir = "dummy"  # for metadata
+
+    def is_latent_cacheable(self) -> bool:
+        return False
+
+    def __len__(self):
+        raise NotImplementedError
+
+    # override to avoid shuffling buckets
+    def set_current_epoch(self, epoch):
+        self.current_epoch = epoch
+
+    def __getitem__(self, idx):
+        r"""
+        The subclass may have image_data for debug_dataset, which is a dict of ImageInfo objects.
+
+        Returns: example like this:
+
+            for i in range(batch_size):
+                image_key = ...  # whatever hashable
+                image_keys.append(image_key)
+
+                image = ...  # PIL Image
+                img_tensor = self.image_transforms(img)
+                images.append(img_tensor)
+
+                caption = ...  # str
+                input_ids = self.get_input_ids(caption)
+                input_ids_list.append(input_ids)
+
+                captions.append(caption)
+
+            images = torch.stack(images, dim=0)
+            input_ids_list = torch.stack(input_ids_list, dim=0)
+            example = {
+                "images": images,
+                "input_ids": input_ids_list,
+                "captions": captions,   # for debug_dataset
+                "latents": None,
+                "image_keys": image_keys,   # for debug_dataset
+                "loss_weights": torch.ones(batch_size, dtype=torch.float32),
+            }
+            return example
+        """
+        raise NotImplementedError
+
+
+def load_arbitrary_dataset(args, tokenizer) -> MinimalDataset:
+    module = ".".join(args.dataset_class.split(".")[:-1])
+    dataset_class = args.dataset_class.split(".")[-1]
+    module = importlib.import_module(module)
+    dataset_class = getattr(module, dataset_class)
+    train_dataset_group: MinimalDataset = dataset_class(tokenizer, args.max_token_length, args.resolution, args.debug_dataset)
+    return train_dataset_group
+
+
+# endregion
+
+# region モジュール入れ替え部
+"""
+高速化のためのモジュール入れ替え
+"""
+
+# FlashAttentionを使うCrossAttention
+# based on https://github.com/lucidrains/memory-efficient-attention-pytorch/blob/main/memory_efficient_attention_pytorch/flash_attention.py
+# LICENSE MIT https://github.com/lucidrains/memory-efficient-attention-pytorch/blob/main/LICENSE
+
+# constants
+
+EPSILON = 1e-6
+
+# helper functions
+
+
+def exists(val):
+    return val is not None
+
+
+def default(val, d):
+    return val if exists(val) else d
+
+
+def model_hash(filename):
+    """Old model hash used by stable-diffusion-webui"""
+    try:
+        with open(filename, "rb") as file:
+            m = hashlib.sha256()
+
+            file.seek(0x100000)
+            m.update(file.read(0x10000))
+            return m.hexdigest()[0:8]
+    except FileNotFoundError:
+        return "NOFILE"
+    except IsADirectoryError:  # Linux?
+        return "IsADirectory"
+    except PermissionError:  # Windows
+        return "IsADirectory"
+
+
+def calculate_sha256(filename):
+    """New model hash used by stable-diffusion-webui"""
+    try:
+        hash_sha256 = hashlib.sha256()
+        blksize = 1024 * 1024
+
+        with open(filename, "rb") as f:
+            for chunk in iter(lambda: f.read(blksize), b""):
+                hash_sha256.update(chunk)
+
+        return hash_sha256.hexdigest()
+    except FileNotFoundError:
+        return "NOFILE"
+    except IsADirectoryError:  # Linux?
+        return "IsADirectory"
+    except PermissionError:  # Windows
+        return "IsADirectory"
+
+
+def precalculate_safetensors_hashes(tensors, metadata):
+    """Precalculate the model hashes needed by sd-webui-additional-networks to
+    save time on indexing the model later."""
+
+    # Because writing user metadata to the file can change the result of
+    # sd_models.model_hash(), only retain the training metadata for purposes of
+    # calculating the hash, as they are meant to be immutable
+    metadata = {k: v for k, v in metadata.items() if k.startswith("ss_")}
+
+    bytes = safetensors.torch.save(tensors, metadata)
+    b = BytesIO(bytes)
+
+    model_hash = addnet_hash_safetensors(b)
+    legacy_hash = addnet_hash_legacy(b)
+    return model_hash, legacy_hash
+
+
+def addnet_hash_legacy(b):
+    """Old model hash used by sd-webui-additional-networks for .safetensors format files"""
+    m = hashlib.sha256()
+
+    b.seek(0x100000)
+    m.update(b.read(0x10000))
+    return m.hexdigest()[0:8]
+
+
+def addnet_hash_safetensors(b):
+    """New model hash used by sd-webui-additional-networks for .safetensors format files"""
+    hash_sha256 = hashlib.sha256()
+    blksize = 1024 * 1024
+
+    b.seek(0)
+    header = b.read(8)
+    n = int.from_bytes(header, "little")
+
+    offset = n + 8
+    b.seek(offset)
+    for chunk in iter(lambda: b.read(blksize), b""):
+        hash_sha256.update(chunk)
+
+    return hash_sha256.hexdigest()
+
+
+def get_git_revision_hash() -> str:
+    try:
+        return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=os.path.dirname(__file__)).decode("ascii").strip()
+    except:
+        return "(unknown)"
+
+
+# def replace_unet_modules(unet: diffusers.models.unet_2d_condition.UNet2DConditionModel, mem_eff_attn, xformers):
+#     replace_attentions_for_hypernetwork()
+#     # unet is not used currently, but it is here for future use
+#     unet.enable_xformers_memory_efficient_attention()
+#     return
+#     if mem_eff_attn:
+#         unet.set_attn_processor(FlashAttnProcessor())
+#     elif xformers:
+#         unet.enable_xformers_memory_efficient_attention()
+
+
+# def replace_unet_cross_attn_to_xformers():
+#     print("CrossAttention.forward has been replaced to enable xformers.")
+#     try:
+#         import xformers.ops
+#     except ImportError:
+#         raise ImportError("No xformers / xformersがインストールされていないようです")
+
+#     def forward_xformers(self, x, context=None, mask=None):
+#         h = self.heads
+#         q_in = self.to_q(x)
+
+#         context = default(context, x)
+#         context = context.to(x.dtype)
+
+#         if hasattr(self, "hypernetwork") and self.hypernetwork is not None:
+#             context_k, context_v = self.hypernetwork.forward(x, context)
+#             context_k = context_k.to(x.dtype)
+#             context_v = context_v.to(x.dtype)
+#         else:
+#             context_k = context
+#             context_v = context
+
+#         k_in = self.to_k(context_k)
+#         v_in = self.to_v(context_v)
+
+#         q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b n h d", h=h), (q_in, k_in, v_in))
+#         del q_in, k_in, v_in
+
+#         q = q.contiguous()
+#         k = k.contiguous()
+#         v = v.contiguous()
+#         out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None)  # 最適なのを選んでくれる
+
+#         out = rearrange(out, "b n h d -> b n (h d)", h=h)
+
+#         # diffusers 0.7.0~
+#         out = self.to_out[0](out)
+#         out = self.to_out[1](out)
+#         return out
+
+
+#     diffusers.models.attention.CrossAttention.forward = forward_xformers
+def replace_unet_modules(unet: UNet2DConditionModel, mem_eff_attn, xformers, sdpa):
+    if mem_eff_attn:
+        print("Enable memory efficient attention for U-Net")
+        unet.set_use_memory_efficient_attention(False, True)
+    elif xformers:
+        print("Enable xformers for U-Net")
+        try:
+            import xformers.ops
+        except ImportError:
+            raise ImportError("No xformers / xformersがインストールされていないようです")
+
+        unet.set_use_memory_efficient_attention(True, False)
+    elif sdpa:
+        print("Enable SDPA for U-Net")
+        unet.set_use_sdpa(True)
+
+
+"""
+def replace_vae_modules(vae: diffusers.models.AutoencoderKL, mem_eff_attn, xformers):
+    # vae is not used currently, but it is here for future use
+    if mem_eff_attn:
+        replace_vae_attn_to_memory_efficient()
+    elif xformers:
+        # とりあえずDiffusersのxformersを使う。AttentionがあるのはMidBlockのみ
+        print("Use Diffusers xformers for VAE")
+        vae.encoder.mid_block.attentions[0].set_use_memory_efficient_attention_xformers(True)
+        vae.decoder.mid_block.attentions[0].set_use_memory_efficient_attention_xformers(True)
+
+
+def replace_vae_attn_to_memory_efficient():
+    print("AttentionBlock.forward has been replaced to FlashAttention (not xformers)")
+    flash_func = FlashAttentionFunction
+
+    def forward_flash_attn(self, hidden_states):
+        print("forward_flash_attn")
+        q_bucket_size = 512
+        k_bucket_size = 1024
+
+        residual = hidden_states
+        batch, channel, height, width = hidden_states.shape
+
+        # norm
+        hidden_states = self.group_norm(hidden_states)
+
+        hidden_states = hidden_states.view(batch, channel, height * width).transpose(1, 2)
+
+        # proj to q, k, v
+        query_proj = self.query(hidden_states)
+        key_proj = self.key(hidden_states)
+        value_proj = self.value(hidden_states)
+
+        query_proj, key_proj, value_proj = map(
+            lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.num_heads), (query_proj, key_proj, value_proj)
+        )
+
+        out = flash_func.apply(query_proj, key_proj, value_proj, None, False, q_bucket_size, k_bucket_size)
+
+        out = rearrange(out, "b h n d -> b n (h d)")
+
+        # compute next hidden_states
+        hidden_states = self.proj_attn(hidden_states)
+        hidden_states = hidden_states.transpose(-1, -2).reshape(batch, channel, height, width)
+
+        # res connect and rescale
+        hidden_states = (hidden_states + residual) / self.rescale_output_factor
+        return hidden_states
+
+    diffusers.models.attention.AttentionBlock.forward = forward_flash_attn
+"""
+
+
+# endregion
+
+
+# region arguments
+
+
+def add_sd_models_arguments(parser: argparse.ArgumentParser):
+    # for pretrained models
+    parser.add_argument("--v2", action="store_true", help="load Stable Diffusion v2.0 model / Stable Diffusion 2.0のモデルを読み込む")
+    parser.add_argument(
+        "--v_parameterization", action="store_true", help="enable v-parameterization training / v-parameterization学習を有効にする"
+    )
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        type=str,
+        default=None,
+        help="pretrained model to train, directory to Diffusers model or StableDiffusion checkpoint / 学習元モデル、Diffusers形式モデルのディレクトリまたはStableDiffusionのckptファイル",
+    )
+    parser.add_argument(
+        "--tokenizer_cache_dir",
+        type=str,
+        default=None,
+        help="directory for caching Tokenizer (for offline training) / Tokenizerをキャッシュするディレクトリ（ネット接続なしでの学習のため）",
+    )
+
+
+def add_optimizer_arguments(parser: argparse.ArgumentParser):
+    parser.add_argument(
+        "--optimizer_type",
+        type=str,
+        default="",
+        help="Optimizer to use / オプティマイザの種類: AdamW (default), AdamW8bit, Lion8bit, Lion, SGDNesterov, SGDNesterov8bit, DAdaptation(DAdaptAdamPreprint), DAdaptAdaGrad, DAdaptAdam, DAdaptAdan, DAdaptAdanIP, DAdaptLion, DAdaptSGD, AdaFactor",
+    )
+
+    # backward compatibility
+    parser.add_argument(
+        "--use_8bit_adam",
+        action="store_true",
+        help="use 8bit AdamW optimizer (requires bitsandbytes) / 8bit Adamオプティマイザを使う（bitsandbytesのインストールが必要）",
+    )
+    parser.add_argument(
+        "--use_lion_optimizer",
+        action="store_true",
+        help="use Lion optimizer (requires lion-pytorch) / Lionオプティマイザを使う（ lion-pytorch のインストールが必要）",
+    )
+
+    parser.add_argument("--learning_rate", type=float, default=2.0e-6, help="learning rate / 学習率")
+    parser.add_argument(
+        "--max_grad_norm", default=1.0, type=float, help="Max gradient norm, 0 for no clipping / 勾配正規化の最大norm、0でclippingを行わない"
+    )
+
+    parser.add_argument(
+        "--optimizer_args",
+        type=str,
+        default=None,
+        nargs="*",
+        help='additional arguments for optimizer (like "weight_decay=0.01 betas=0.9,0.999 ...") / オプティマイザの追加引数（例： "weight_decay=0.01 betas=0.9,0.999 ..."）',
+    )
+
+    parser.add_argument("--lr_scheduler_type", type=str, default="", help="custom scheduler module / 使用するスケジューラ")
+    parser.add_argument(
+        "--lr_scheduler_args",
+        type=str,
+        default=None,
+        nargs="*",
+        help='additional arguments for scheduler (like "T_max=100") / スケジューラの追加引数（例： "T_max100"）',
+    )
+
+    parser.add_argument(
+        "--lr_scheduler",
+        type=str,
+        default="constant",
+        help="scheduler to use for learning rate / 学習率のスケジューラ: linear, cosine, cosine_with_restarts, polynomial, constant (default), constant_with_warmup, adafactor",
+    )
+    parser.add_argument(
+        "--lr_warmup_steps",
+        type=int,
+        default=0,
+        help="Number of steps for the warmup in the lr scheduler (default is 0) / 学習率のスケジューラをウォームアップするステップ数（デフォルト0）",
+    )
+    parser.add_argument(
+        "--lr_scheduler_num_cycles",
+        type=int,
+        default=1,
+        help="Number of restarts for cosine scheduler with restarts / cosine with restartsスケジューラでのリスタート回数",
+    )
+    parser.add_argument(
+        "--lr_scheduler_power",
+        type=float,
+        default=1,
+        help="Polynomial power for polynomial scheduler / polynomialスケジューラでのpolynomial power",
+    )
+
+
+def add_training_arguments(parser: argparse.ArgumentParser, support_dreambooth: bool):
+    parser.add_argument("--output_dir", type=str, default=None, help="directory to output trained model / 学習後のモデル出力先ディレクトリ")
+    parser.add_argument("--output_name", type=str, default=None, help="base name of trained model file / 学習後のモデルの拡張子を除くファイル名")
+    parser.add_argument(
+        "--huggingface_repo_id", type=str, default=None, help="huggingface repo name to upload / huggingfaceにアップロードするリポジトリ名"
+    )
+    parser.add_argument(
+        "--huggingface_repo_type", type=str, default=None, help="huggingface repo type to upload / huggingfaceにアップロードするリポジトリの種類"
+    )
+    parser.add_argument(
+        "--huggingface_path_in_repo",
+        type=str,
+        default=None,
+        help="huggingface model path to upload files / huggingfaceにアップロードするファイルのパス",
+    )
+    parser.add_argument("--huggingface_token", type=str, default=None, help="huggingface token / huggingfaceのトークン")
+    parser.add_argument(
+        "--huggingface_repo_visibility",
+        type=str,
+        default=None,
+        help="huggingface repository visibility ('public' for public, 'private' or None for private) / huggingfaceにアップロードするリポジトリの公開設定（'public'で公開、'private'またはNoneで非公開）",
+    )
+    parser.add_argument(
+        "--save_state_to_huggingface", action="store_true", help="save state to huggingface / huggingfaceにstateを保存する"
+    )
+    parser.add_argument(
+        "--resume_from_huggingface",
+        action="store_true",
+        help="resume from huggingface (ex: --resume {repo_id}/{path_in_repo}:{revision}:{repo_type}) / huggingfaceから学習を再開する(例: --resume {repo_id}/{path_in_repo}:{revision}:{repo_type})",
+    )
+    parser.add_argument(
+        "--async_upload",
+        action="store_true",
+        help="upload to huggingface asynchronously / huggingfaceに非同期でアップロードする",
+    )
+    parser.add_argument(
+        "--save_precision",
+        type=str,
+        default=None,
+        choices=[None, "float", "fp16", "bf16"],
+        help="precision in saving / 保存時に精度を変更して保存する",
+    )
+    parser.add_argument(
+        "--save_every_n_epochs", type=int, default=None, help="save checkpoint every N epochs / 学習中のモデルを指定エポックごとに保存する"
+    )
+    parser.add_argument(
+        "--save_every_n_steps", type=int, default=None, help="save checkpoint every N steps / 学習中のモデルを指定ステップごとに保存する"
+    )
+    parser.add_argument(
+        "--save_n_epoch_ratio",
+        type=int,
+        default=None,
+        help="save checkpoint N epoch ratio (for example 5 means save at least 5 files total) / 学習中のモデルを指定のエポック割合で保存する（たとえば5を指定すると最低5個のファイルが保存される）",
+    )
+    parser.add_argument(
+        "--save_last_n_epochs",
+        type=int,
+        default=None,
+        help="save last N checkpoints when saving every N epochs (remove older checkpoints) / 指定エポックごとにモデルを保存するとき最大Nエポック保存する（古いチェックポイントは削除する）",
+    )
+    parser.add_argument(
+        "--save_last_n_epochs_state",
+        type=int,
+        default=None,
+        help="save last N checkpoints of state (overrides the value of --save_last_n_epochs)/ 最大Nエポックstateを保存する（--save_last_n_epochsの指定を上書きする）",
+    )
+    parser.add_argument(
+        "--save_last_n_steps",
+        type=int,
+        default=None,
+        help="save checkpoints until N steps elapsed (remove older checkpoints if N steps elapsed) / 指定ステップごとにモデルを保存するとき、このステップ数経過するまで保存する（このステップ数経過したら削除する）",
+    )
+    parser.add_argument(
+        "--save_last_n_steps_state",
+        type=int,
+        default=None,
+        help="save states until N steps elapsed (remove older states if N steps elapsed, overrides --save_last_n_steps) / 指定ステップごとにstateを保存するとき、このステップ数経過するまで保存する（このステップ数経過したら削除する。--save_last_n_stepsを上書きする）",
+    )
+    parser.add_argument(
+        "--save_state",
+        action="store_true",
+        help="save training state additionally (including optimizer states etc.) / optimizerなど学習状態も含めたstateを追加で保存する",
+    )
+    parser.add_argument("--resume", type=str, default=None, help="saved state to resume training / 学習再開するモデルのstate")
+
+    parser.add_argument("--train_batch_size", type=int, default=1, help="batch size for training / 学習時のバッチサイズ")
+    parser.add_argument(
+        "--max_token_length",
+        type=int,
+        default=None,
+        choices=[None, 150, 225],
+        help="max token length of text encoder (default for 75, 150 or 225) / text encoderのトークンの最大長（未指定で75、150または225が指定可）",
+    )
+    parser.add_argument(
+        "--mem_eff_attn",
+        action="store_true",
+        help="use memory efficient attention for CrossAttention / CrossAttentionに省メモリ版attentionを使う",
+    )
+    parser.add_argument("--xformers", action="store_true", help="use xformers for CrossAttention / CrossAttentionにxformersを使う")
+    parser.add_argument(
+        "--sdpa",
+        action="store_true",
+        help="use sdpa for CrossAttention (requires PyTorch 2.0) / CrossAttentionにsdpaを使う（PyTorch 2.0が必要）",
+    )
+    parser.add_argument(
+        "--vae", type=str, default=None, help="path to checkpoint of vae to replace / VAEを入れ替える場合、VAEのcheckpointファイルまたはディレクトリ"
+    )
+
+    parser.add_argument("--max_train_steps", type=int, default=1600, help="training steps / 学習ステップ数")
+    parser.add_argument(
+        "--max_train_epochs",
+        type=int,
+        default=None,
+        help="training epochs (overrides max_train_steps) / 学習エポック数（max_train_stepsを上書きします）",
+    )
+    parser.add_argument(
+        "--max_data_loader_n_workers",
+        type=int,
+        default=8,
+        help="max num workers for DataLoader (lower is less main RAM usage, faster epoch start and slower data loading) / DataLoaderの最大プロセス数（小さい値ではメインメモリの使用量が減りエポック間の待ち時間が減りますが、データ読み込みは遅くなります）",
+    )
+    parser.add_argument(
+        "--persistent_data_loader_workers",
+        action="store_true",
+        help="persistent DataLoader workers (useful for reduce time gap between epoch, but may use more memory) / DataLoader のワーカーを持続させる (エポック間の時間差を少なくするのに有効だが、より多くのメモリを消費する可能性がある)",
+    )
+    parser.add_argument("--seed", type=int, default=None, help="random seed for training / 学習時の乱数のseed")
+    parser.add_argument(
+        "--gradient_checkpointing", action="store_true", help="enable gradient checkpointing / grandient checkpointingを有効にする"
+    )
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Number of updates steps to accumulate before performing a backward/update pass / 学習時に逆伝播をする前に勾配を合計するステップ数",
+    )
+    parser.add_argument(
+        "--mixed_precision", type=str, default="no", choices=["no", "fp16", "bf16"], help="use mixed precision / 混合精度を使う場合、その精度"
+    )
+    parser.add_argument("--full_fp16", action="store_true", help="fp16 training including gradients / 勾配も含めてfp16で学習する")
+    parser.add_argument("--full_bf16", action="store_true", help="bf16 training including gradients / 勾配も含めてbf16で学習する")
+    parser.add_argument(
+        "--clip_skip",
+        type=int,
+        default=None,
+        help="use output of nth layer from back of text encoder (n>=1) / text encoderの後ろからn番目の層の出力を用いる（nは1以上）",
+    )
+    parser.add_argument(
+        "--logging_dir",
+        type=str,
+        default=None,
+        help="enable logging and output TensorBoard log to this directory / ログ出力を有効にしてこのディレクトリにTensorBoard用のログを出力する",
+    )
+    parser.add_argument(
+        "--log_with",
+        type=str,
+        default=None,
+        choices=["tensorboard", "wandb", "all"],
+        help="what logging tool(s) to use (if 'all', TensorBoard and WandB are both used) / ログ出力に使用するツール (allを指定するとTensorBoardとWandBの両方が使用される)",
+    )
+    parser.add_argument("--log_prefix", type=str, default=None, help="add prefix for each log directory / ログディレクトリ名の先頭に追加する文字列")
+    parser.add_argument(
+        "--log_tracker_name",
+        type=str,
+        default=None,
+        help="name of tracker to use for logging, default is script-specific default name / ログ出力に使用するtrackerの名前、省略時はスクリプトごとのデフォルト名",
+    )
+    parser.add_argument(
+        "--wandb_api_key",
+        type=str,
+        default=None,
+        help="specify WandB API key to log in before starting training (optional). / WandB APIキーを指定して学習開始前にログインする（オプション）",
+    )
+    parser.add_argument(
+        "--noise_offset",
+        type=float,
+        default=None,
+        help="enable noise offset with this value (if enabled, around 0.1 is recommended) / Noise offsetを有効にしてこの値を設定する（有効にする場合は0.1程度を推奨）",
+    )
+    parser.add_argument(
+        "--multires_noise_iterations",
+        type=int,
+        default=None,
+        help="enable multires noise with this number of iterations (if enabled, around 6-10 is recommended) / Multires noiseを有効にしてこのイテレーション数を設定する（有効にする場合は6-10程度を推奨）",
+    )
+    # parser.add_argument(
+    #     "--perlin_noise",
+    #     type=int,
+    #     default=None,
+    #     help="enable perlin noise and set the octaves / perlin noiseを有効にしてoctavesをこの値に設定する",
+    # )
+    parser.add_argument(
+        "--multires_noise_discount",
+        type=float,
+        default=0.3,
+        help="set discount value for multires noise (has no effect without --multires_noise_iterations) / Multires noiseのdiscount値を設定する（--multires_noise_iterations指定時のみ有効）",
+    )
+    parser.add_argument(
+        "--adaptive_noise_scale",
+        type=float,
+        default=None,
+        help="add `latent mean absolute value * this value` to noise_offset (disabled if None, default) / latentの平均値の絶対値 * この値をnoise_offsetに加算する（Noneの場合は無効、デフォルト）",
+    )
+    parser.add_argument(
+        "--min_timestep",
+        type=int,
+        default=None,
+        help="set minimum time step for U-Net training (0~999, default is 0) / U-Net学習時のtime stepの最小値を設定する（0~999で指定、省略時はデフォルト値(0)） ",
+    )
+    parser.add_argument(
+        "--max_timestep",
+        type=int,
+        default=None,
+        help="set maximum time step for U-Net training (1~1000, default is 1000) / U-Net学習時のtime stepの最大値を設定する（1~1000で指定、省略時はデフォルト値(1000)）",
+    )
+
+    parser.add_argument(
+        "--lowram",
+        action="store_true",
+        help="enable low RAM optimization. e.g. load models to VRAM instead of RAM (for machines which have bigger VRAM than RAM such as Colab and Kaggle) / メインメモリが少ない環境向け最適化を有効にする。たとえばVRAMにモデルを読み込むなど（ColabやKaggleなどRAMに比べてVRAMが多い環境向け）",
+    )
+
+    parser.add_argument(
+        "--sample_every_n_steps", type=int, default=None, help="generate sample images every N steps / 学習中のモデルで指定ステップごとにサンプル出力する"
+    )
+    parser.add_argument(
+        "--sample_every_n_epochs",
+        type=int,
+        default=None,
+        help="generate sample images every N epochs (overwrites n_steps) / 学習中のモデルで指定エポックごとにサンプル出力する（ステップ数指定を上書きします）",
+    )
+    parser.add_argument(
+        "--sample_prompts", type=str, default=None, help="file for prompts to generate sample images / 学習中モデルのサンプル出力用プロンプトのファイル"
+    )
+    parser.add_argument(
+        "--sample_sampler",
+        type=str,
+        default="ddim",
+        choices=[
+            "ddim",
+            "pndm",
+            "lms",
+            "euler",
+            "euler_a",
+            "heun",
+            "dpm_2",
+            "dpm_2_a",
+            "dpmsolver",
+            "dpmsolver++",
+            "dpmsingle",
+            "k_lms",
+            "k_euler",
+            "k_euler_a",
+            "k_dpm_2",
+            "k_dpm_2_a",
+        ],
+        help=f"sampler (scheduler) type for sample images / サンプル出力時のサンプラー（スケジューラ）の種類",
+    )
+
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        default=None,
+        help="using .toml instead of args to pass hyperparameter / ハイパーパラメータを引数ではなく.tomlファイルで渡す",
+    )
+    parser.add_argument(
+        "--output_config", action="store_true", help="output command line args to given .toml file / 引数を.tomlファイルに出力する"
+    )
+
+    if support_dreambooth:
+        # DreamBooth training
+        parser.add_argument(
+            "--prior_loss_weight", type=float, default=1.0, help="loss weight for regularization images / 正則化画像のlossの重み"
+        )
+
+
+def verify_training_args(args: argparse.Namespace):
+    if args.v_parameterization and not args.v2:
+        print("v_parameterization should be with v2 / v1でv_parameterizationを使用することは想定されていません")
+    if args.v2 and args.clip_skip is not None:
+        print("v2 with clip_skip will be unexpected / v2でclip_skipを使用することは想定されていません")
+
+    if args.cache_latents_to_disk and not args.cache_latents:
+        args.cache_latents = True
+        print(
+            "cache_latents_to_disk is enabled, so cache_latents is also enabled / cache_latents_to_diskが有効なため、cache_latentsを有効にします"
+        )
+
+    # noise_offset, perlin_noise, multires_noise_iterations cannot be enabled at the same time
+    # Listを使って数えてもいいけど並べてしまえ
+    if args.noise_offset is not None and args.multires_noise_iterations is not None:
+        raise ValueError(
+            "noise_offset and multires_noise_iterations cannot be enabled at the same time / noise_offsetとmultires_noise_iterationsを同時に有効にできません"
+        )
+    # if args.noise_offset is not None and args.perlin_noise is not None:
+    #     raise ValueError("noise_offset and perlin_noise cannot be enabled at the same time / noise_offsetとperlin_noiseは同時に有効にできません")
+    # if args.perlin_noise is not None and args.multires_noise_iterations is not None:
+    #     raise ValueError(
+    #         "perlin_noise and multires_noise_iterations cannot be enabled at the same time / perlin_noiseとmultires_noise_iterationsを同時に有効にできません"
+    #     )
+
+    if args.adaptive_noise_scale is not None and args.noise_offset is None:
+        raise ValueError("adaptive_noise_scale requires noise_offset / adaptive_noise_scaleを使用するにはnoise_offsetが必要です")
+
+    if args.scale_v_pred_loss_like_noise_pred and not args.v_parameterization:
+        raise ValueError(
+            "scale_v_pred_loss_like_noise_pred can be enabled only with v_parameterization / scale_v_pred_loss_like_noise_predはv_parameterizationが有効なときのみ有効にできます"
+        )
+
+
+def add_dataset_arguments(
+    parser: argparse.ArgumentParser, support_dreambooth: bool, support_caption: bool, support_caption_dropout: bool
+):
+    # dataset common
+    parser.add_argument("--train_data_dir", type=str, default=None, help="directory for train images / 学習画像データのディレクトリ")
+    parser.add_argument(
+        "--shuffle_caption", action="store_true", help="shuffle comma-separated caption / コンマで区切られたcaptionの各要素をshuffleする"
+    )
+    parser.add_argument(
+        "--caption_extension", type=str, default=".caption", help="extension of caption files / 読み込むcaptionファイルの拡張子"
+    )
+    parser.add_argument(
+        "--caption_extention",
+        type=str,
+        default=None,
+        help="extension of caption files (backward compatibility) / 読み込むcaptionファイルの拡張子（スペルミスを残してあります）",
+    )
+    parser.add_argument(
+        "--keep_tokens",
+        type=int,
+        default=0,
+        help="keep heading N tokens when shuffling caption tokens (token means comma separated strings) / captionのシャッフル時に、先頭からこの個数のトークンをシャッフルしないで残す（トークンはカンマ区切りの各部分を意味する）",
+    )
+    parser.add_argument("--color_aug", action="store_true", help="enable weak color augmentation / 学習時に色合いのaugmentationを有効にする")
+    parser.add_argument("--flip_aug", action="store_true", help="enable horizontal flip augmentation / 学習時に左右反転のaugmentationを有効にする")
+    parser.add_argument(
+        "--face_crop_aug_range",
+        type=str,
+        default=None,
+        help="enable face-centered crop augmentation and its range (e.g. 2.0,4.0) / 学習時に顔を中心とした切り出しaugmentationを有効にするときは倍率を指定する（例：2.0,4.0）",
+    )
+    parser.add_argument(
+        "--random_crop",
+        action="store_true",
+        help="enable random crop (for style training in face-centered crop augmentation) / ランダムな切り出しを有効にする（顔を中心としたaugmentationを行うときに画風の学習用に指定する）",
+    )
+    parser.add_argument(
+        "--debug_dataset", action="store_true", help="show images for debugging (do not train) / デバッグ用に学習データを画面表示する（学習は行わない）"
+    )
+    parser.add_argument(
+        "--resolution",
+        type=str,
+        default=None,
+        help="resolution in training ('size' or 'width,height') / 学習時の画像解像度（'サイズ'指定、または'幅,高さ'指定）",
+    )
+    parser.add_argument(
+        "--cache_latents",
+        action="store_true",
+        help="cache latents to main memory to reduce VRAM usage (augmentations must be disabled) / VRAM削減のためにlatentをメインメモリにcacheする（augmentationは使用不可） ",
+    )
+    parser.add_argument("--vae_batch_size", type=int, default=1, help="batch size for caching latents / latentのcache時のバッチサイズ")
+    parser.add_argument(
+        "--cache_latents_to_disk",
+        action="store_true",
+        help="cache latents to disk to reduce VRAM usage (augmentations must be disabled) / VRAM削減のためにlatentをディスクにcacheする（augmentationは使用不可）",
+    )
+    parser.add_argument(
+        "--enable_bucket", action="store_true", help="enable buckets for multi aspect ratio training / 複数解像度学習のためのbucketを有効にする"
+    )
+    parser.add_argument("--min_bucket_reso", type=int, default=256, help="minimum resolution for buckets / bucketの最小解像度")
+    parser.add_argument("--max_bucket_reso", type=int, default=1024, help="maximum resolution for buckets / bucketの最大解像度")
+    parser.add_argument(
+        "--bucket_reso_steps",
+        type=int,
+        default=64,
+        help="steps of resolution for buckets, divisible by 8 is recommended / bucketの解像度の単位、8で割り切れる値を推奨します",
+    )
+    parser.add_argument(
+        "--bucket_no_upscale", action="store_true", help="make bucket for each image without upscaling / 画像を拡大せずbucketを作成します"
+    )
+
+    parser.add_argument(
+        "--token_warmup_min",
+        type=int,
+        default=1,
+        help="start learning at N tags (token means comma separated strinfloatgs) / タグ数をN個から増やしながら学習する",
+    )
+    parser.add_argument(
+        "--token_warmup_step",
+        type=float,
+        default=0,
+        help="tag length reaches maximum on N steps (or N*max_train_steps if N<1) / N（N<1ならN*max_train_steps）ステップでタグ長が最大になる。デフォルトは0（最初から最大）",
+    )
+
+    parser.add_argument(
+        "--dataset_class",
+        type=str,
+        default=None,
+        help="dataset class for arbitrary dataset (package.module.Class) / 任意のデータセットを用いるときのクラス名 (package.module.Class)",
+    )
+
+    if support_caption_dropout:
+        # Textual Inversion はcaptionのdropoutをsupportしない
+        # いわゆるtensorのDropoutと紛らわしいのでprefixにcaptionを付けておく　every_n_epochsは他と平仄を合わせてdefault Noneに
+        parser.add_argument(
+            "--caption_dropout_rate", type=float, default=0.0, help="Rate out dropout caption(0.0~1.0) / captionをdropoutする割合"
+        )
+        parser.add_argument(
+            "--caption_dropout_every_n_epochs",
+            type=int,
+            default=0,
+            help="Dropout all captions every N epochs / captionを指定エポックごとにdropoutする",
+        )
+        parser.add_argument(
+            "--caption_tag_dropout_rate",
+            type=float,
+            default=0.0,
+            help="Rate out dropout comma separated tokens(0.0~1.0) / カンマ区切りのタグをdropoutする割合",
+        )
+
+    if support_dreambooth:
+        # DreamBooth dataset
+        parser.add_argument("--reg_data_dir", type=str, default=None, help="directory for regularization images / 正則化画像データのディレクトリ")
+
+    if support_caption:
+        # caption dataset
+        parser.add_argument("--in_json", type=str, default=None, help="json metadata for dataset / データセットのmetadataのjsonファイル")
+        parser.add_argument(
+            "--dataset_repeats", type=int, default=1, help="repeat dataset when training with captions / キャプションでの学習時にデータセットを繰り返す回数"
+        )
+
+
+def add_sd_saving_arguments(parser: argparse.ArgumentParser):
+    parser.add_argument(
+        "--save_model_as",
+        type=str,
+        default=None,
+        choices=[None, "ckpt", "safetensors", "diffusers", "diffusers_safetensors"],
+        help="format to save the model (default is same to original) / モデル保存時の形式（未指定時は元モデルと同じ）",
+    )
+    parser.add_argument(
+        "--use_safetensors",
+        action="store_true",
+        help="use safetensors format to save (if save_model_as is not specified) / checkpoint、モデルをsafetensors形式で保存する（save_model_as未指定時）",
+    )
+
+
+def read_config_from_file(args: argparse.Namespace, parser: argparse.ArgumentParser):
+    if not args.config_file:
+        return args
+
+    config_path = args.config_file + ".toml" if not args.config_file.endswith(".toml") else args.config_file
+
+    if args.output_config:
+        # check if config file exists
+        if os.path.exists(config_path):
+            print(f"Config file already exists. Aborting... / 出力先の設定ファイルが既に存在します: {config_path}")
+            exit(1)
+
+        # convert args to dictionary
+        args_dict = vars(args)
+
+        # remove unnecessary keys
+        for key in ["config_file", "output_config", "wandb_api_key"]:
+            if key in args_dict:
+                del args_dict[key]
+
+        # get default args from parser
+        default_args = vars(parser.parse_args([]))
+
+        # remove default values: cannot use args_dict.items directly because it will be changed during iteration
+        for key, value in list(args_dict.items()):
+            if key in default_args and value == default_args[key]:
+                del args_dict[key]
+
+        # convert Path to str in dictionary
+        for key, value in args_dict.items():
+            if isinstance(value, pathlib.Path):
+                args_dict[key] = str(value)
+
+        # convert to toml and output to file
+        with open(config_path, "w") as f:
+            toml.dump(args_dict, f)
+
+        print(f"Saved config file / 設定ファイルを保存しました: {config_path}")
+        exit(0)
+
+    if not os.path.exists(config_path):
+        print(f"{config_path} not found.")
+        exit(1)
+
+    print(f"Loading settings from {config_path}...")
+    with open(config_path, "r") as f:
+        config_dict = toml.load(f)
+
+    # combine all sections into one
+    ignore_nesting_dict = {}
+    for section_name, section_dict in config_dict.items():
+        # if value is not dict, save key and value as is
+        if not isinstance(section_dict, dict):
+            ignore_nesting_dict[section_name] = section_dict
+            continue
+
+        # if value is dict, save all key and value into one dict
+        for key, value in section_dict.items():
+            ignore_nesting_dict[key] = value
+
+    config_args = argparse.Namespace(**ignore_nesting_dict)
+    args = parser.parse_args(namespace=config_args)
+    args.config_file = os.path.splitext(args.config_file)[0]
+    print(args.config_file)
+
+    return args
+
+
+# endregion
+
+# region utils
+
+
+def resume_from_local_or_hf_if_specified(accelerator, args):
+    if not args.resume:
+        return
+
+    if not args.resume_from_huggingface:
+        print(f"resume training from local state: {args.resume}")
+        accelerator.load_state(args.resume)
+        return
+
+    print(f"resume training from huggingface state: {args.resume}")
+    repo_id = args.resume.split("/")[0] + "/" + args.resume.split("/")[1]
+    path_in_repo = "/".join(args.resume.split("/")[2:])
+    revision = None
+    repo_type = None
+    if ":" in path_in_repo:
+        divided = path_in_repo.split(":")
+        if len(divided) == 2:
+            path_in_repo, revision = divided
+            repo_type = "model"
+        else:
+            path_in_repo, revision, repo_type = divided
+    print(f"Downloading state from huggingface: {repo_id}/{path_in_repo}@{revision}")
+
+    list_files = huggingface_util.list_dir(
+        repo_id=repo_id,
+        subfolder=path_in_repo,
+        revision=revision,
+        token=args.huggingface_token,
+        repo_type=repo_type,
+    )
+
+    async def download(filename) -> str:
+        def task():
+            return hf_hub_download(
+                repo_id=repo_id,
+                filename=filename,
+                revision=revision,
+                repo_type=repo_type,
+                token=args.huggingface_token,
+            )
+
+        return await asyncio.get_event_loop().run_in_executor(None, task)
+
+    loop = asyncio.get_event_loop()
+    results = loop.run_until_complete(asyncio.gather(*[download(filename=filename.rfilename) for filename in list_files]))
+    if len(results) == 0:
+        raise ValueError("No files found in the specified repo id/path/revision / 指定されたリポジトリID/パス/リビジョンにファイルが見つかりませんでした")
+    dirname = os.path.dirname(results[0])
+    accelerator.load_state(dirname)
+
+
+def get_optimizer(args, trainable_params):
+    # "Optimizer to use: AdamW, AdamW8bit, Lion, SGDNesterov, SGDNesterov8bit, Lion8bit, DAdaptation(DAdaptAdamPreprint), DAdaptAdaGrad, DAdaptAdam, DAdaptAdan, DAdaptAdanIP, DAdaptLion, DAdaptSGD, Adafactor"
+
+    optimizer_type = args.optimizer_type
+    if args.use_8bit_adam:
+        assert (
+            not args.use_lion_optimizer
+        ), "both option use_8bit_adam and use_lion_optimizer are specified / use_8bit_adamとuse_lion_optimizerの両方のオプションが指定されています"
+        assert (
+            optimizer_type is None or optimizer_type == ""
+        ), "both option use_8bit_adam and optimizer_type are specified / use_8bit_adamとoptimizer_typeの両方のオプションが指定されています"
+        optimizer_type = "AdamW8bit"
+
+    elif args.use_lion_optimizer:
+        assert (
+            optimizer_type is None or optimizer_type == ""
+        ), "both option use_lion_optimizer and optimizer_type are specified / use_lion_optimizerとoptimizer_typeの両方のオプションが指定されています"
+        optimizer_type = "Lion"
+
+    if optimizer_type is None or optimizer_type == "":
+        optimizer_type = "AdamW"
+    optimizer_type = optimizer_type.lower()
+
+    # 引数を分解する
+    optimizer_kwargs = {}
+    if args.optimizer_args is not None and len(args.optimizer_args) > 0:
+        for arg in args.optimizer_args:
+            key, value = arg.split("=")
+            value = ast.literal_eval(value)
+
+            # value = value.split(",")
+            # for i in range(len(value)):
+            #     if value[i].lower() == "true" or value[i].lower() == "false":
+            #         value[i] = value[i].lower() == "true"
+            #     else:
+            #         value[i] = ast.float(value[i])
+            # if len(value) == 1:
+            #     value = value[0]
+            # else:
+            #     value = tuple(value)
+
+            optimizer_kwargs[key] = value
+    # print("optkwargs:", optimizer_kwargs)
+
+    lr = args.learning_rate
+
+    if optimizer_type == "AdamW8bit".lower():
+        try:
+            import bitsandbytes as bnb
+        except ImportError:
+            raise ImportError("No bitsand bytes / bitsandbytesがインストールされていないようです")
+        print(f"use 8-bit AdamW optimizer | {optimizer_kwargs}")
+        optimizer_class = bnb.optim.AdamW8bit
+        optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs)
+
+    elif optimizer_type == "SGDNesterov8bit".lower():
+        try:
+            import bitsandbytes as bnb
+        except ImportError:
+            raise ImportError("No bitsand bytes / bitsandbytesがインストールされていないようです")
+        print(f"use 8-bit SGD with Nesterov optimizer | {optimizer_kwargs}")
+        if "momentum" not in optimizer_kwargs:
+            print(
+                f"8-bit SGD with Nesterov must be with momentum, set momentum to 0.9 / 8-bit SGD with Nesterovはmomentum指定が必須のため0.9に設定します"
+            )
+            optimizer_kwargs["momentum"] = 0.9
+
+        optimizer_class = bnb.optim.SGD8bit
+        optimizer = optimizer_class(trainable_params, lr=lr, nesterov=True, **optimizer_kwargs)
+
+    elif optimizer_type == "Lion".lower():
+        try:
+            import lion_pytorch
+        except ImportError:
+            raise ImportError("No lion_pytorch / lion_pytorch がインストールされていないようです")
+        print(f"use Lion optimizer | {optimizer_kwargs}")
+        optimizer_class = lion_pytorch.Lion
+        optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs)
+
+    elif optimizer_type == "Lion8bit".lower():
+        try:
+            import bitsandbytes as bnb
+        except ImportError:
+            raise ImportError("No bitsandbytes / bitsandbytesがインストールされていないようです")
+
+        print(f"use 8-bit Lion optimizer | {optimizer_kwargs}")
+        try:
+            optimizer_class = bnb.optim.Lion8bit
+        except AttributeError:
+            raise AttributeError(
+                "No Lion8bit. The version of bitsandbytes installed seems to be old. Please install 0.38.0 or later. / Lion8bitが定義されていません。インストールされているbitsandbytesのバージョンが古いようです。0.38.0以上をインストールしてください"
+            )
+
+        optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs)
+
+    elif optimizer_type == "SGDNesterov".lower():
+        print(f"use SGD with Nesterov optimizer | {optimizer_kwargs}")
+        if "momentum" not in optimizer_kwargs:
+            print(f"SGD with Nesterov must be with momentum, set momentum to 0.9 / SGD with Nesterovはmomentum指定が必須のため0.9に設定します")
+            optimizer_kwargs["momentum"] = 0.9
+
+        optimizer_class = torch.optim.SGD
+        optimizer = optimizer_class(trainable_params, lr=lr, nesterov=True, **optimizer_kwargs)
+
+    elif optimizer_type.startswith("DAdapt".lower()) or optimizer_type == "Prodigy".lower():
+        # check lr and lr_count, and print warning
+        actual_lr = lr
+        lr_count = 1
+        if type(trainable_params) == list and type(trainable_params[0]) == dict:
+            lrs = set()
+            actual_lr = trainable_params[0].get("lr", actual_lr)
+            for group in trainable_params:
+                lrs.add(group.get("lr", actual_lr))
+            lr_count = len(lrs)
+
+        if actual_lr <= 0.1:
+            print(
+                f"learning rate is too low. If using D-Adaptation or Prodigy, set learning rate around 1.0 / 学習率が低すぎるようです。D-AdaptationまたはProdigyの使用時は1.0前後の値を指定してください: lr={actual_lr}"
+            )
+            print("recommend option: lr=1.0 / 推奨は1.0です")
+        if lr_count > 1:
+            print(
+                f"when multiple learning rates are specified with dadaptation (e.g. for Text Encoder and U-Net), only the first one will take effect / D-AdaptationまたはProdigyで複数の学習率を指定した場合（Text EncoderとU-Netなど）、最初の学習率のみが有効になります: lr={actual_lr}"
+            )
+
+        if optimizer_type.startswith("DAdapt".lower()):
+            # DAdaptation family
+            # check dadaptation is installed
+            try:
+                import dadaptation
+                import dadaptation.experimental as experimental
+            except ImportError:
+                raise ImportError("No dadaptation / dadaptation がインストールされていないようです")
+
+            # set optimizer
+            if optimizer_type == "DAdaptation".lower() or optimizer_type == "DAdaptAdamPreprint".lower():
+                optimizer_class = experimental.DAdaptAdamPreprint
+                print(f"use D-Adaptation AdamPreprint optimizer | {optimizer_kwargs}")
+            elif optimizer_type == "DAdaptAdaGrad".lower():
+                optimizer_class = dadaptation.DAdaptAdaGrad
+                print(f"use D-Adaptation AdaGrad optimizer | {optimizer_kwargs}")
+            elif optimizer_type == "DAdaptAdam".lower():
+                optimizer_class = dadaptation.DAdaptAdam
+                print(f"use D-Adaptation Adam optimizer | {optimizer_kwargs}")
+            elif optimizer_type == "DAdaptAdan".lower():
+                optimizer_class = dadaptation.DAdaptAdan
+                print(f"use D-Adaptation Adan optimizer | {optimizer_kwargs}")
+            elif optimizer_type == "DAdaptAdanIP".lower():
+                optimizer_class = experimental.DAdaptAdanIP
+                print(f"use D-Adaptation AdanIP optimizer | {optimizer_kwargs}")
+            elif optimizer_type == "DAdaptLion".lower():
+                optimizer_class = dadaptation.DAdaptLion
+                print(f"use D-Adaptation Lion optimizer | {optimizer_kwargs}")
+            elif optimizer_type == "DAdaptSGD".lower():
+                optimizer_class = dadaptation.DAdaptSGD
+                print(f"use D-Adaptation SGD optimizer | {optimizer_kwargs}")
+            else:
+                raise ValueError(f"Unknown optimizer type: {optimizer_type}")
+
+            optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs)
+        else:
+            # Prodigy
+            # check Prodigy is installed
+            try:
+                import prodigyopt
+            except ImportError:
+                raise ImportError("No Prodigy / Prodigy がインストールされていないようです")
+
+            print(f"use Prodigy optimizer | {optimizer_kwargs}")
+            optimizer_class = prodigyopt.Prodigy
+            optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs)
+
+    elif optimizer_type == "Adafactor".lower():
+        # 引数を確認して適宜補正する
+        if "relative_step" not in optimizer_kwargs:
+            optimizer_kwargs["relative_step"] = True  # default
+        if not optimizer_kwargs["relative_step"] and optimizer_kwargs.get("warmup_init", False):
+            print(f"set relative_step to True because warmup_init is True / warmup_initがTrueのためrelative_stepをTrueにします")
+            optimizer_kwargs["relative_step"] = True
+        print(f"use Adafactor optimizer | {optimizer_kwargs}")
+
+        if optimizer_kwargs["relative_step"]:
+            print(f"relative_step is true / relative_stepがtrueです")
+            if lr != 0.0:
+                print(f"learning rate is used as initial_lr / 指定したlearning rateはinitial_lrとして使用されます")
+            args.learning_rate = None
+
+            # trainable_paramsがgroupだった時の処理：lrを削除する
+            if type(trainable_params) == list and type(trainable_params[0]) == dict:
+                has_group_lr = False
+                for group in trainable_params:
+                    p = group.pop("lr", None)
+                    has_group_lr = has_group_lr or (p is not None)
+
+                if has_group_lr:
+                    # 一応argsを無効にしておく TODO 依存関係が逆転してるのであまり望ましくない
+                    print(f"unet_lr and text_encoder_lr are ignored / unet_lrとtext_encoder_lrは無視されます")
+                    args.unet_lr = None
+                    args.text_encoder_lr = None
+
+            if args.lr_scheduler != "adafactor":
+                print(f"use adafactor_scheduler / スケジューラにadafactor_schedulerを使用します")
+            args.lr_scheduler = f"adafactor:{lr}"  # ちょっと微妙だけど
+
+            lr = None
+        else:
+            if args.max_grad_norm != 0.0:
+                print(
+                    f"because max_grad_norm is set, clip_grad_norm is enabled. consider set to 0 / max_grad_normが設定されているためclip_grad_normが有効になります。0に設定して無効にしたほうがいいかもしれません"
+                )
+            if args.lr_scheduler != "constant_with_warmup":
+                print(f"constant_with_warmup will be good / スケジューラはconstant_with_warmupが良いかもしれません")
+            if optimizer_kwargs.get("clip_threshold", 1.0) != 1.0:
+                print(f"clip_threshold=1.0 will be good / clip_thresholdは1.0が良いかもしれません")
+
+        optimizer_class = transformers.optimization.Adafactor
+        optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs)
+
+    elif optimizer_type == "AdamW".lower():
+        print(f"use AdamW optimizer | {optimizer_kwargs}")
+        optimizer_class = torch.optim.AdamW
+        optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs)
+
+    else:
+        # 任意のoptimizerを使う
+        optimizer_type = args.optimizer_type  # lowerでないやつ（微妙）
+        print(f"use {optimizer_type} | {optimizer_kwargs}")
+        if "." not in optimizer_type:
+            optimizer_module = torch.optim
+        else:
+            values = optimizer_type.split(".")
+            optimizer_module = importlib.import_module(".".join(values[:-1]))
+            optimizer_type = values[-1]
+
+        optimizer_class = getattr(optimizer_module, optimizer_type)
+        optimizer = optimizer_class(trainable_params, lr=lr, **optimizer_kwargs)
+
+    optimizer_name = optimizer_class.__module__ + "." + optimizer_class.__name__
+    optimizer_args = ",".join([f"{k}={v}" for k, v in optimizer_kwargs.items()])
+
+    return optimizer_name, optimizer_args, optimizer
+
+
+# Monkeypatch newer get_scheduler() function overridng current version of diffusers.optimizer.get_scheduler
+# code is taken from https://github.com/huggingface/diffusers diffusers.optimizer, commit d87cc15977b87160c30abaace3894e802ad9e1e6
+# Which is a newer release of diffusers than currently packaged with sd-scripts
+# This code can be removed when newer diffusers version (v0.12.1 or greater) is tested and implemented to sd-scripts
+
+
+def get_scheduler_fix(args, optimizer: Optimizer, num_processes: int):
+    """
+    Unified API to get any scheduler from its name.
+    """
+    name = args.lr_scheduler
+    num_warmup_steps: Optional[int] = args.lr_warmup_steps
+    num_training_steps = args.max_train_steps * num_processes * args.gradient_accumulation_steps
+    num_cycles = args.lr_scheduler_num_cycles
+    power = args.lr_scheduler_power
+
+    lr_scheduler_kwargs = {}  # get custom lr_scheduler kwargs
+    if args.lr_scheduler_args is not None and len(args.lr_scheduler_args) > 0:
+        for arg in args.lr_scheduler_args:
+            key, value = arg.split("=")
+
+            value = ast.literal_eval(value)
+            # value = value.split(",")
+            # for i in range(len(value)):
+            #     if value[i].lower() == "true" or value[i].lower() == "false":
+            #         value[i] = value[i].lower() == "true"
+            #     else:
+            #         value[i] = ast.literal_eval(value[i])
+            # if len(value) == 1:
+            #     value = value[0]
+            # else:
+            #     value = list(value)  # some may use list?
+
+            lr_scheduler_kwargs[key] = value
+
+    def wrap_check_needless_num_warmup_steps(return_vals):
+        if num_warmup_steps is not None and num_warmup_steps != 0:
+            raise ValueError(f"{name} does not require `num_warmup_steps`. Set None or 0.")
+        return return_vals
+
+    # using any lr_scheduler from other library
+    if args.lr_scheduler_type:
+        lr_scheduler_type = args.lr_scheduler_type
+        print(f"use {lr_scheduler_type} | {lr_scheduler_kwargs} as lr_scheduler")
+        if "." not in lr_scheduler_type:  # default to use torch.optim
+            lr_scheduler_module = torch.optim.lr_scheduler
+        else:
+            values = lr_scheduler_type.split(".")
+            lr_scheduler_module = importlib.import_module(".".join(values[:-1]))
+            lr_scheduler_type = values[-1]
+        lr_scheduler_class = getattr(lr_scheduler_module, lr_scheduler_type)
+        lr_scheduler = lr_scheduler_class(optimizer, **lr_scheduler_kwargs)
+        return wrap_check_needless_num_warmup_steps(lr_scheduler)
+
+    if name.startswith("adafactor"):
+        assert (
+            type(optimizer) == transformers.optimization.Adafactor
+        ), f"adafactor scheduler must be used with Adafactor optimizer / adafactor schedulerはAdafactorオプティマイザと同時に使ってください"
+        initial_lr = float(name.split(":")[1])
+        # print("adafactor scheduler init lr", initial_lr)
+        return wrap_check_needless_num_warmup_steps(transformers.optimization.AdafactorSchedule(optimizer, initial_lr))
+
+    name = SchedulerType(name)
+    schedule_func = TYPE_TO_SCHEDULER_FUNCTION[name]
+    if name == SchedulerType.CONSTANT:
+        return wrap_check_needless_num_warmup_steps(schedule_func(optimizer))
+
+    # All other schedulers require `num_warmup_steps`
+    if num_warmup_steps is None:
+        raise ValueError(f"{name} requires `num_warmup_steps`, please provide that argument.")
+
+    if name == SchedulerType.CONSTANT_WITH_WARMUP:
+        return schedule_func(optimizer, num_warmup_steps=num_warmup_steps)
+
+    # All other schedulers require `num_training_steps`
+    if num_training_steps is None:
+        raise ValueError(f"{name} requires `num_training_steps`, please provide that argument.")
+
+    if name == SchedulerType.COSINE_WITH_RESTARTS:
+        return schedule_func(
+            optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps, num_cycles=num_cycles
+        )
+
+    if name == SchedulerType.POLYNOMIAL:
+        return schedule_func(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps, power=power)
+
+    return schedule_func(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps)
+
+
+def prepare_dataset_args(args: argparse.Namespace, support_metadata: bool):
+    # backward compatibility
+    if args.caption_extention is not None:
+        args.caption_extension = args.caption_extention
+        args.caption_extention = None
+
+    # assert args.resolution is not None, f"resolution is required / resolution（解像度）を指定してください"
+    if args.resolution is not None:
+        args.resolution = tuple([int(r) for r in args.resolution.split(",")])
+        if len(args.resolution) == 1:
+            args.resolution = (args.resolution[0], args.resolution[0])
+        assert (
+            len(args.resolution) == 2
+        ), f"resolution must be 'size' or 'width,height' / resolution（解像度）は'サイズ'または'幅','高さ'で指定してください: {args.resolution}"
+
+    if args.face_crop_aug_range is not None:
+        args.face_crop_aug_range = tuple([float(r) for r in args.face_crop_aug_range.split(",")])
+        assert (
+            len(args.face_crop_aug_range) == 2 and args.face_crop_aug_range[0] <= args.face_crop_aug_range[1]
+        ), f"face_crop_aug_range must be two floats / face_crop_aug_rangeは'下限,上限'で指定してください: {args.face_crop_aug_range}"
+    else:
+        args.face_crop_aug_range = None
+
+    if support_metadata:
+        if args.in_json is not None and (args.color_aug or args.random_crop):
+            print(
+                f"latents in npz is ignored when color_aug or random_crop is True / color_augまたはrandom_cropを有効にした場合、npzファイルのlatentsは無視されます"
+            )
+
+
+def load_tokenizer(args: argparse.Namespace):
+    print("prepare tokenizer")
+    original_path = V2_STABLE_DIFFUSION_PATH if args.v2 else TOKENIZER_PATH
+
+    tokenizer: CLIPTokenizer = None
+    if args.tokenizer_cache_dir:
+        local_tokenizer_path = os.path.join(args.tokenizer_cache_dir, original_path.replace("/", "_"))
+        if os.path.exists(local_tokenizer_path):
+            print(f"load tokenizer from cache: {local_tokenizer_path}")
+            tokenizer = CLIPTokenizer.from_pretrained(local_tokenizer_path)  # same for v1 and v2
+
+    if tokenizer is None:
+        if args.v2:
+            tokenizer = CLIPTokenizer.from_pretrained(original_path, subfolder="tokenizer")
+        else:
+            tokenizer = CLIPTokenizer.from_pretrained(original_path)
+
+    if hasattr(args, "max_token_length") and args.max_token_length is not None:
+        print(f"update token length: {args.max_token_length}")
+
+    if args.tokenizer_cache_dir and not os.path.exists(local_tokenizer_path):
+        print(f"save Tokenizer to cache: {local_tokenizer_path}")
+        tokenizer.save_pretrained(local_tokenizer_path)
+
+    return tokenizer
+
+
+def prepare_accelerator(args: argparse.Namespace):
+    if args.logging_dir is None:
+        logging_dir = None
+    else:
+        log_prefix = "" if args.log_prefix is None else args.log_prefix
+        logging_dir = args.logging_dir + "/" + log_prefix + time.strftime("%Y%m%d%H%M%S", time.localtime())
+
+    if args.log_with is None:
+        if logging_dir is not None:
+            log_with = "tensorboard"
+        else:
+            log_with = None
+    else:
+        log_with = args.log_with
+        if log_with in ["tensorboard", "all"]:
+            if logging_dir is None:
+                raise ValueError("logging_dir is required when log_with is tensorboard / Tensorboardを使う場合、logging_dirを指定してください")
+        if log_with in ["wandb", "all"]:
+            try:
+                import wandb
+            except ImportError:
+                raise ImportError("No wandb / wandb がインストールされていないようです")
+            if logging_dir is not None:
+                os.makedirs(logging_dir, exist_ok=True)
+                os.environ["WANDB_DIR"] = logging_dir
+            if args.wandb_api_key is not None:
+                wandb.login(key=args.wandb_api_key)
+
+    accelerator = Accelerator(
+        gradient_accumulation_steps=args.gradient_accumulation_steps,
+        mixed_precision=args.mixed_precision,
+        log_with=log_with,
+        project_dir=logging_dir,
+    )
+    return accelerator
+
+
+def prepare_dtype(args: argparse.Namespace):
+    weight_dtype = torch.float32
+    if args.mixed_precision == "fp16":
+        weight_dtype = torch.float16
+    elif args.mixed_precision == "bf16":
+        weight_dtype = torch.bfloat16
+
+    save_dtype = None
+    if args.save_precision == "fp16":
+        save_dtype = torch.float16
+    elif args.save_precision == "bf16":
+        save_dtype = torch.bfloat16
+    elif args.save_precision == "float":
+        save_dtype = torch.float32
+
+    return weight_dtype, save_dtype
+
+
+def _load_target_model(args: argparse.Namespace, weight_dtype, device="cpu", unet_use_linear_projection_in_v2=False):
+    name_or_path = args.pretrained_model_name_or_path
+    name_or_path = os.readlink(name_or_path) if os.path.islink(name_or_path) else name_or_path
+    load_stable_diffusion_format = os.path.isfile(name_or_path)  # determine SD or Diffusers
+    if load_stable_diffusion_format:
+        print(f"load StableDiffusion checkpoint: {name_or_path}")
+        text_encoder, vae, unet = model_util.load_models_from_stable_diffusion_checkpoint(
+            args.v2, name_or_path, device, unet_use_linear_projection_in_v2=unet_use_linear_projection_in_v2
+        )
+    else:
+        # Diffusers model is loaded to CPU
+        print(f"load Diffusers pretrained models: {name_or_path}")
+        try:
+            pipe = StableDiffusionPipeline.from_pretrained(name_or_path, tokenizer=None, safety_checker=None)
+        except EnvironmentError as ex:
+            print(
+                f"model is not found as a file or in Hugging Face, perhaps file name is wrong? / 指定したモデル名のファイル、またはHugging Faceのモデルが見つかりません。ファイル名が誤っているかもしれません: {name_or_path}"
+            )
+            raise ex
+        text_encoder = pipe.text_encoder
+        vae = pipe.vae
+        unet = pipe.unet
+        del pipe
+
+        # Diffusers U-Net to original U-Net
+        # TODO *.ckpt/*.safetensorsのv2と同じ形式にここで変換すると良さそう
+        # print(f"unet config: {unet.config}")
+        original_unet = UNet2DConditionModel(
+            unet.config.sample_size,
+            unet.config.attention_head_dim,
+            unet.config.cross_attention_dim,
+            unet.config.use_linear_projection,
+            unet.config.upcast_attention,
+        )
+        original_unet.load_state_dict(unet.state_dict())
+        unet = original_unet
+        print("U-Net converted to original U-Net")
+
+    # VAEを読み込む
+    if args.vae is not None:
+        vae = model_util.load_vae(args.vae, weight_dtype)
+        print("additional VAE loaded")
+
+    return text_encoder, vae, unet, load_stable_diffusion_format
+
+
+# TODO remove this function in the future
+def transform_if_model_is_DDP(text_encoder, unet, network=None):
+    # Transform text_encoder, unet and network from DistributedDataParallel
+    return (model.module if type(model) == DDP else model for model in [text_encoder, unet, network] if model is not None)
+
+
+def transform_models_if_DDP(models):
+    # Transform text_encoder, unet and network from DistributedDataParallel
+    return [model.module if type(model) == DDP else model for model in models if model is not None]
+
+
+def load_target_model(args, weight_dtype, accelerator, unet_use_linear_projection_in_v2=False):
+    # load models for each process
+    for pi in range(accelerator.state.num_processes):
+        if pi == accelerator.state.local_process_index:
+            print(f"loading model for process {accelerator.state.local_process_index}/{accelerator.state.num_processes}")
+
+            text_encoder, vae, unet, load_stable_diffusion_format = _load_target_model(
+                args,
+                weight_dtype,
+                accelerator.device if args.lowram else "cpu",
+                unet_use_linear_projection_in_v2=unet_use_linear_projection_in_v2,
+            )
+
+            # work on low-ram device
+            if args.lowram:
+                text_encoder.to(accelerator.device)
+                unet.to(accelerator.device)
+                vae.to(accelerator.device)
+
+            gc.collect()
+            torch.cuda.empty_cache()
+        accelerator.wait_for_everyone()
+
+    text_encoder, unet = transform_if_model_is_DDP(text_encoder, unet)
+
+    return text_encoder, vae, unet, load_stable_diffusion_format
+
+
+def patch_accelerator_for_fp16_training(accelerator):
+    org_unscale_grads = accelerator.scaler._unscale_grads_
+
+    def _unscale_grads_replacer(optimizer, inv_scale, found_inf, allow_fp16):
+        return org_unscale_grads(optimizer, inv_scale, found_inf, True)
+
+    accelerator.scaler._unscale_grads_ = _unscale_grads_replacer
+
+
+def get_hidden_states(args: argparse.Namespace, input_ids, tokenizer, text_encoder, weight_dtype=None):
+    # with no_token_padding, the length is not max length, return result immediately
+    if input_ids.size()[-1] != tokenizer.model_max_length:
+        return text_encoder(input_ids)[0]
+
+    # input_ids: b,n,77
+    b_size = input_ids.size()[0]
+    input_ids = input_ids.reshape((-1, tokenizer.model_max_length))  # batch_size*3, 77
+
+    if args.clip_skip is None:
+        encoder_hidden_states = text_encoder(input_ids)[0]
+    else:
+        enc_out = text_encoder(input_ids, output_hidden_states=True, return_dict=True)
+        encoder_hidden_states = enc_out["hidden_states"][-args.clip_skip]
+        encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
+
+    # bs*3, 77, 768 or 1024
+    encoder_hidden_states = encoder_hidden_states.reshape((b_size, -1, encoder_hidden_states.shape[-1]))
+
+    if args.max_token_length is not None:
+        if args.v2:
+            # v2: <BOS>...<EOS> <PAD> ... の三連を <BOS>...<EOS> <PAD> ... へ戻す　正直この実装でいいのかわからん
+            states_list = [encoder_hidden_states[:, 0].unsqueeze(1)]  # <BOS>
+            for i in range(1, args.max_token_length, tokenizer.model_max_length):
+                chunk = encoder_hidden_states[:, i : i + tokenizer.model_max_length - 2]  # <BOS> の後から 最後の前まで
+                if i > 0:
+                    for j in range(len(chunk)):
+                        if input_ids[j, 1] == tokenizer.eos_token:  # 空、つまり <BOS> <EOS> <PAD> ...のパターン
+                            chunk[j, 0] = chunk[j, 1]  # 次の <PAD> の値をコピーする
+                states_list.append(chunk)  # <BOS> の後から <EOS> の前まで
+            states_list.append(encoder_hidden_states[:, -1].unsqueeze(1))  # <EOS> か <PAD> のどちらか
+            encoder_hidden_states = torch.cat(states_list, dim=1)
+        else:
+            # v1: <BOS>...<EOS> の三連を <BOS>...<EOS> へ戻す
+            states_list = [encoder_hidden_states[:, 0].unsqueeze(1)]  # <BOS>
+            for i in range(1, args.max_token_length, tokenizer.model_max_length):
+                states_list.append(encoder_hidden_states[:, i : i + tokenizer.model_max_length - 2])  # <BOS> の後から <EOS> の前まで
+            states_list.append(encoder_hidden_states[:, -1].unsqueeze(1))  # <EOS>
+            encoder_hidden_states = torch.cat(states_list, dim=1)
+
+    if weight_dtype is not None:
+        # this is required for additional network training
+        encoder_hidden_states = encoder_hidden_states.to(weight_dtype)
+
+    return encoder_hidden_states
+
+
+def default_if_none(value, default):
+    return default if value is None else value
+
+
+def get_epoch_ckpt_name(args: argparse.Namespace, ext: str, epoch_no: int):
+    model_name = default_if_none(args.output_name, DEFAULT_EPOCH_NAME)
+    return EPOCH_FILE_NAME.format(model_name, epoch_no) + ext
+
+
+def get_step_ckpt_name(args: argparse.Namespace, ext: str, step_no: int):
+    model_name = default_if_none(args.output_name, DEFAULT_STEP_NAME)
+    return STEP_FILE_NAME.format(model_name, step_no) + ext
+
+
+def get_last_ckpt_name(args: argparse.Namespace, ext: str):
+    model_name = default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME)
+    return model_name + ext
+
+
+def get_remove_epoch_no(args: argparse.Namespace, epoch_no: int):
+    if args.save_last_n_epochs is None:
+        return None
+
+    remove_epoch_no = epoch_no - args.save_every_n_epochs * args.save_last_n_epochs
+    if remove_epoch_no < 0:
+        return None
+    return remove_epoch_no
+
+
+def get_remove_step_no(args: argparse.Namespace, step_no: int):
+    if args.save_last_n_steps is None:
+        return None
+
+    # last_n_steps前のstep_noから、save_every_n_stepsの倍数のstep_noを計算して削除する
+    # save_every_n_steps=10, save_last_n_steps=30の場合、50step目には30step分残し、10step目を削除する
+    remove_step_no = step_no - args.save_last_n_steps - 1
+    remove_step_no = remove_step_no - (remove_step_no % args.save_every_n_steps)
+    if remove_step_no < 0:
+        return None
+    return remove_step_no
+
+
+# epochとstepの保存、メタデータにepoch/stepが含まれ引数が同じになるため、統合している
+# on_epoch_end: Trueならepoch終了時、Falseならstep経過時
+def save_sd_model_on_epoch_end_or_stepwise(
+    args: argparse.Namespace,
+    on_epoch_end: bool,
+    accelerator,
+    src_path: str,
+    save_stable_diffusion_format: bool,
+    use_safetensors: bool,
+    save_dtype: torch.dtype,
+    epoch: int,
+    num_train_epochs: int,
+    global_step: int,
+    text_encoder,
+    unet,
+    vae,
+):
+    def sd_saver(ckpt_file, epoch_no, global_step):
+        model_util.save_stable_diffusion_checkpoint(
+            args.v2, ckpt_file, text_encoder, unet, src_path, epoch_no, global_step, save_dtype, vae
+        )
+
+    def diffusers_saver(out_dir):
+        model_util.save_diffusers_checkpoint(
+            args.v2, out_dir, text_encoder, unet, src_path, vae=vae, use_safetensors=use_safetensors
+        )
+
+    save_sd_model_on_epoch_end_or_stepwise_common(
+        args,
+        on_epoch_end,
+        accelerator,
+        save_stable_diffusion_format,
+        use_safetensors,
+        epoch,
+        num_train_epochs,
+        global_step,
+        sd_saver,
+        diffusers_saver,
+    )
+
+
+def save_sd_model_on_epoch_end_or_stepwise_common(
+    args: argparse.Namespace,
+    on_epoch_end: bool,
+    accelerator,
+    save_stable_diffusion_format: bool,
+    use_safetensors: bool,
+    epoch: int,
+    num_train_epochs: int,
+    global_step: int,
+    sd_saver,
+    diffusers_saver,
+):
+    if on_epoch_end:
+        epoch_no = epoch + 1
+        saving = epoch_no % args.save_every_n_epochs == 0 and epoch_no < num_train_epochs
+        if not saving:
+            return
+
+        model_name = default_if_none(args.output_name, DEFAULT_EPOCH_NAME)
+        remove_no = get_remove_epoch_no(args, epoch_no)
+    else:
+        # 保存するか否かは呼び出し側で判断済み
+
+        model_name = default_if_none(args.output_name, DEFAULT_STEP_NAME)
+        epoch_no = epoch  # 例: 最初のepochの途中で保存したら0になる、SDモデルに保存される
+        remove_no = get_remove_step_no(args, global_step)
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    if save_stable_diffusion_format:
+        ext = ".safetensors" if use_safetensors else ".ckpt"
+
+        if on_epoch_end:
+            ckpt_name = get_epoch_ckpt_name(args, ext, epoch_no)
+        else:
+            ckpt_name = get_step_ckpt_name(args, ext, global_step)
+
+        ckpt_file = os.path.join(args.output_dir, ckpt_name)
+        print(f"\nsaving checkpoint: {ckpt_file}")
+        sd_saver(ckpt_file, epoch_no, global_step)
+
+        if args.huggingface_repo_id is not None:
+            huggingface_util.upload(args, ckpt_file, "/" + ckpt_name)
+
+        # remove older checkpoints
+        if remove_no is not None:
+            if on_epoch_end:
+                remove_ckpt_name = get_epoch_ckpt_name(args, ext, remove_no)
+            else:
+                remove_ckpt_name = get_step_ckpt_name(args, ext, remove_no)
+
+            remove_ckpt_file = os.path.join(args.output_dir, remove_ckpt_name)
+            if os.path.exists(remove_ckpt_file):
+                print(f"removing old checkpoint: {remove_ckpt_file}")
+                os.remove(remove_ckpt_file)
+
+    else:
+        if on_epoch_end:
+            out_dir = os.path.join(args.output_dir, EPOCH_DIFFUSERS_DIR_NAME.format(model_name, epoch_no))
+        else:
+            out_dir = os.path.join(args.output_dir, STEP_DIFFUSERS_DIR_NAME.format(model_name, global_step))
+
+        print(f"\nsaving model: {out_dir}")
+        diffusers_saver(out_dir)
+
+        if args.huggingface_repo_id is not None:
+            huggingface_util.upload(args, out_dir, "/" + model_name)
+
+        # remove older checkpoints
+        if remove_no is not None:
+            if on_epoch_end:
+                remove_out_dir = os.path.join(args.output_dir, EPOCH_DIFFUSERS_DIR_NAME.format(model_name, remove_no))
+            else:
+                remove_out_dir = os.path.join(args.output_dir, STEP_DIFFUSERS_DIR_NAME.format(model_name, remove_no))
+
+            if os.path.exists(remove_out_dir):
+                print(f"removing old model: {remove_out_dir}")
+                shutil.rmtree(remove_out_dir)
+
+    if args.save_state:
+        if on_epoch_end:
+            save_and_remove_state_on_epoch_end(args, accelerator, epoch_no)
+        else:
+            save_and_remove_state_stepwise(args, accelerator, global_step)
+
+
+def save_and_remove_state_on_epoch_end(args: argparse.Namespace, accelerator, epoch_no):
+    model_name = default_if_none(args.output_name, DEFAULT_EPOCH_NAME)
+
+    print(f"\nsaving state at epoch {epoch_no}")
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    state_dir = os.path.join(args.output_dir, EPOCH_STATE_NAME.format(model_name, epoch_no))
+    accelerator.save_state(state_dir)
+    if args.save_state_to_huggingface:
+        print("uploading state to huggingface.")
+        huggingface_util.upload(args, state_dir, "/" + EPOCH_STATE_NAME.format(model_name, epoch_no))
+
+    last_n_epochs = args.save_last_n_epochs_state if args.save_last_n_epochs_state else args.save_last_n_epochs
+    if last_n_epochs is not None:
+        remove_epoch_no = epoch_no - args.save_every_n_epochs * last_n_epochs
+        state_dir_old = os.path.join(args.output_dir, EPOCH_STATE_NAME.format(model_name, remove_epoch_no))
+        if os.path.exists(state_dir_old):
+            print(f"removing old state: {state_dir_old}")
+            shutil.rmtree(state_dir_old)
+
+
+def save_and_remove_state_stepwise(args: argparse.Namespace, accelerator, step_no):
+    model_name = default_if_none(args.output_name, DEFAULT_STEP_NAME)
+
+    print(f"\nsaving state at step {step_no}")
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    state_dir = os.path.join(args.output_dir, STEP_STATE_NAME.format(model_name, step_no))
+    accelerator.save_state(state_dir)
+    if args.save_state_to_huggingface:
+        print("uploading state to huggingface.")
+        huggingface_util.upload(args, state_dir, "/" + STEP_STATE_NAME.format(model_name, step_no))
+
+    last_n_steps = args.save_last_n_steps_state if args.save_last_n_steps_state else args.save_last_n_steps
+    if last_n_steps is not None:
+        # last_n_steps前のstep_noから、save_every_n_stepsの倍数のstep_noを計算して削除する
+        remove_step_no = step_no - last_n_steps - 1
+        remove_step_no = remove_step_no - (remove_step_no % args.save_every_n_steps)
+
+        if remove_step_no > 0:
+            state_dir_old = os.path.join(args.output_dir, STEP_STATE_NAME.format(model_name, remove_step_no))
+            if os.path.exists(state_dir_old):
+                print(f"removing old state: {state_dir_old}")
+                shutil.rmtree(state_dir_old)
+
+
+def save_state_on_train_end(args: argparse.Namespace, accelerator):
+    model_name = default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME)
+
+    print("\nsaving last state.")
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    state_dir = os.path.join(args.output_dir, LAST_STATE_NAME.format(model_name))
+    accelerator.save_state(state_dir)
+
+    if args.save_state_to_huggingface:
+        print("uploading last state to huggingface.")
+        huggingface_util.upload(args, state_dir, "/" + LAST_STATE_NAME.format(model_name))
+
+
+def save_sd_model_on_train_end(
+    args: argparse.Namespace,
+    src_path: str,
+    save_stable_diffusion_format: bool,
+    use_safetensors: bool,
+    save_dtype: torch.dtype,
+    epoch: int,
+    global_step: int,
+    text_encoder,
+    unet,
+    vae,
+):
+    def sd_saver(ckpt_file, epoch_no, global_step):
+        model_util.save_stable_diffusion_checkpoint(
+            args.v2, ckpt_file, text_encoder, unet, src_path, epoch_no, global_step, save_dtype, vae
+        )
+
+    def diffusers_saver(out_dir):
+        model_util.save_diffusers_checkpoint(
+            args.v2, out_dir, text_encoder, unet, src_path, vae=vae, use_safetensors=use_safetensors
+        )
+
+    save_sd_model_on_train_end_common(
+        args, save_stable_diffusion_format, use_safetensors, epoch, global_step, sd_saver, diffusers_saver
+    )
+
+
+def save_sd_model_on_train_end_common(
+    args: argparse.Namespace,
+    save_stable_diffusion_format: bool,
+    use_safetensors: bool,
+    epoch: int,
+    global_step: int,
+    sd_saver,
+    diffusers_saver,
+):
+    model_name = default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME)
+
+    if save_stable_diffusion_format:
+        os.makedirs(args.output_dir, exist_ok=True)
+
+        ckpt_name = model_name + (".safetensors" if use_safetensors else ".ckpt")
+        ckpt_file = os.path.join(args.output_dir, ckpt_name)
+
+        print(f"save trained model as StableDiffusion checkpoint to {ckpt_file}")
+        sd_saver(ckpt_file, epoch, global_step)
+
+        if args.huggingface_repo_id is not None:
+            huggingface_util.upload(args, ckpt_file, "/" + ckpt_name, force_sync_upload=True)
+    else:
+        out_dir = os.path.join(args.output_dir, model_name)
+        os.makedirs(out_dir, exist_ok=True)
+
+        print(f"save trained model as Diffusers to {out_dir}")
+        diffusers_saver(out_dir)
+
+        if args.huggingface_repo_id is not None:
+            huggingface_util.upload(args, out_dir, "/" + model_name, force_sync_upload=True)
+
+
+def get_noise_noisy_latents_and_timesteps(args, noise_scheduler, latents):
+    # Sample noise that we'll add to the latents
+    noise = torch.randn_like(latents, device=latents.device)
+    if args.noise_offset:
+        noise = custom_train_functions.apply_noise_offset(latents, noise, args.noise_offset, args.adaptive_noise_scale)
+    elif args.multires_noise_iterations:
+        noise = custom_train_functions.pyramid_noise_like(
+            noise, latents.device, args.multires_noise_iterations, args.multires_noise_discount
+        )
+
+    # Sample a random timestep for each image
+    b_size = latents.shape[0]
+    min_timestep = 0 if args.min_timestep is None else args.min_timestep
+    max_timestep = noise_scheduler.config.num_train_timesteps if args.max_timestep is None else args.max_timestep
+
+    timesteps = torch.randint(min_timestep, max_timestep, (b_size,), device=latents.device)
+    timesteps = timesteps.long()
+
+    # Add noise to the latents according to the noise magnitude at each timestep
+    # (this is the forward diffusion process)
+    noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps)
+
+    return noise, noisy_latents, timesteps
+
+
+# scheduler:
+SCHEDULER_LINEAR_START = 0.00085
+SCHEDULER_LINEAR_END = 0.0120
+SCHEDULER_TIMESTEPS = 1000
+SCHEDLER_SCHEDULE = "scaled_linear"
+
+
+def sample_images(*args, **kwargs):
+    return sample_images_common(StableDiffusionLongPromptWeightingPipeline, *args, **kwargs)
+
+
+def sample_images_common(
+    pipe_class,
+    accelerator,
+    args: argparse.Namespace,
+    epoch,
+    steps,
+    device,
+    vae,
+    tokenizer,
+    text_encoder,
+    unet,
+    prompt_replacement=None,
+    controlnet=None,
+):
+    """
+    StableDiffusionLongPromptWeightingPipelineの改造版を使うようにしたので、clip skipおよびプロンプトの重みづけに対応した
+    """
+    if args.sample_every_n_steps is None and args.sample_every_n_epochs is None:
+        return
+    if args.sample_every_n_epochs is not None:
+        # sample_every_n_steps は無視する
+        if epoch is None or epoch % args.sample_every_n_epochs != 0:
+            return
+    else:
+        if steps % args.sample_every_n_steps != 0 or epoch is not None:  # steps is not divisible or end of epoch
+            return
+
+    print(f"\ngenerating sample images at step / サンプル画像生成 ステップ: {steps}")
+    if not os.path.isfile(args.sample_prompts):
+        print(f"No prompt file / プロンプトファイルがありません: {args.sample_prompts}")
+        return
+
+    org_vae_device = vae.device  # CPUにいるはず
+    vae.to(device)
+
+    # read prompts
+
+    # with open(args.sample_prompts, "rt", encoding="utf-8") as f:
+    #     prompts = f.readlines()
+
+    if args.sample_prompts.endswith(".txt"):
+        with open(args.sample_prompts, "r", encoding="utf-8") as f:
+            lines = f.readlines()
+        prompts = [line.strip() for line in lines if len(line.strip()) > 0 and line[0] != "#"]
+    elif args.sample_prompts.endswith(".toml"):
+        with open(args.sample_prompts, "r", encoding="utf-8") as f:
+            data = toml.load(f)
+        prompts = [dict(**data["prompt"], **subset) for subset in data["prompt"]["subset"]]
+    elif args.sample_prompts.endswith(".json"):
+        with open(args.sample_prompts, "r", encoding="utf-8") as f:
+            prompts = json.load(f)
+
+    # schedulerを用意する
+    sched_init_args = {}
+    if args.sample_sampler == "ddim":
+        scheduler_cls = DDIMScheduler
+    elif args.sample_sampler == "ddpm":  # ddpmはおかしくなるのでoptionから外してある
+        scheduler_cls = DDPMScheduler
+    elif args.sample_sampler == "pndm":
+        scheduler_cls = PNDMScheduler
+    elif args.sample_sampler == "lms" or args.sample_sampler == "k_lms":
+        scheduler_cls = LMSDiscreteScheduler
+    elif args.sample_sampler == "euler" or args.sample_sampler == "k_euler":
+        scheduler_cls = EulerDiscreteScheduler
+    elif args.sample_sampler == "euler_a" or args.sample_sampler == "k_euler_a":
+        scheduler_cls = EulerAncestralDiscreteScheduler
+    elif args.sample_sampler == "dpmsolver" or args.sample_sampler == "dpmsolver++":
+        scheduler_cls = DPMSolverMultistepScheduler
+        sched_init_args["algorithm_type"] = args.sample_sampler
+    elif args.sample_sampler == "dpmsingle":
+        scheduler_cls = DPMSolverSinglestepScheduler
+    elif args.sample_sampler == "heun":
+        scheduler_cls = HeunDiscreteScheduler
+    elif args.sample_sampler == "dpm_2" or args.sample_sampler == "k_dpm_2":
+        scheduler_cls = KDPM2DiscreteScheduler
+    elif args.sample_sampler == "dpm_2_a" or args.sample_sampler == "k_dpm_2_a":
+        scheduler_cls = KDPM2AncestralDiscreteScheduler
+    else:
+        scheduler_cls = DDIMScheduler
+
+    if args.v_parameterization:
+        sched_init_args["prediction_type"] = "v_prediction"
+
+    scheduler = scheduler_cls(
+        num_train_timesteps=SCHEDULER_TIMESTEPS,
+        beta_start=SCHEDULER_LINEAR_START,
+        beta_end=SCHEDULER_LINEAR_END,
+        beta_schedule=SCHEDLER_SCHEDULE,
+        **sched_init_args,
+    )
+
+    # clip_sample=Trueにする
+    if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is False:
+        # print("set clip_sample to True")
+        scheduler.config.clip_sample = True
+
+    pipeline = pipe_class(
+        text_encoder=text_encoder,
+        vae=vae,
+        unet=unet,
+        tokenizer=tokenizer,
+        scheduler=scheduler,
+        safety_checker=None,
+        feature_extractor=None,
+        requires_safety_checker=False,
+        clip_skip=args.clip_skip,
+    )
+    pipeline.to(device)
+
+    save_dir = args.output_dir + "/sample"
+    os.makedirs(save_dir, exist_ok=True)
+
+    rng_state = torch.get_rng_state()
+    cuda_rng_state = torch.cuda.get_rng_state() if torch.cuda.is_available() else None
+
+    with torch.no_grad():
+        # with accelerator.autocast():
+        for i, prompt in enumerate(prompts):
+            if not accelerator.is_main_process:
+                continue
+
+            if isinstance(prompt, dict):
+                negative_prompt = prompt.get("negative_prompt")
+                sample_steps = prompt.get("sample_steps", 30)
+                width = prompt.get("width", 512)
+                height = prompt.get("height", 512)
+                scale = prompt.get("scale", 7.5)
+                seed = prompt.get("seed")
+                controlnet_image = prompt.get("controlnet_image")
+                prompt = prompt.get("prompt")
+            else:
+                # prompt = prompt.strip()
+                # if len(prompt) == 0 or prompt[0] == "#":
+                #     continue
+
+                # subset of gen_img_diffusers
+                prompt_args = prompt.split(" --")
+                prompt = prompt_args[0]
+                negative_prompt = None
+                sample_steps = 30
+                width = height = 512
+                scale = 7.5
+                seed = None
+                controlnet_image = None
+                for parg in prompt_args:
+                    try:
+                        m = re.match(r"w (\d+)", parg, re.IGNORECASE)
+                        if m:
+                            width = int(m.group(1))
+                            continue
+
+                        m = re.match(r"h (\d+)", parg, re.IGNORECASE)
+                        if m:
+                            height = int(m.group(1))
+                            continue
+
+                        m = re.match(r"d (\d+)", parg, re.IGNORECASE)
+                        if m:
+                            seed = int(m.group(1))
+                            continue
+
+                        m = re.match(r"s (\d+)", parg, re.IGNORECASE)
+                        if m:  # steps
+                            sample_steps = max(1, min(1000, int(m.group(1))))
+                            continue
+
+                        m = re.match(r"l ([\d\.]+)", parg, re.IGNORECASE)
+                        if m:  # scale
+                            scale = float(m.group(1))
+                            continue
+
+                        m = re.match(r"n (.+)", parg, re.IGNORECASE)
+                        if m:  # negative prompt
+                            negative_prompt = m.group(1)
+                            continue
+
+                        m = re.match(r"cn (.+)", parg, re.IGNORECASE)
+                        if m:  # negative prompt
+                            controlnet_image = m.group(1)
+                            continue
+
+                    except ValueError as ex:
+                        print(f"Exception in parsing / 解析エラー: {parg}")
+                        print(ex)
+
+            if seed is not None:
+                torch.manual_seed(seed)
+                torch.cuda.manual_seed(seed)
+
+            if prompt_replacement is not None:
+                prompt = prompt.replace(prompt_replacement[0], prompt_replacement[1])
+                if negative_prompt is not None:
+                    negative_prompt = negative_prompt.replace(prompt_replacement[0], prompt_replacement[1])
+
+            if controlnet_image is not None:
+                controlnet_image = Image.open(controlnet_image).convert("RGB")
+                controlnet_image = controlnet_image.resize((width, height), Image.LANCZOS)
+
+            height = max(64, height - height % 8)  # round to divisible by 8
+            width = max(64, width - width % 8)  # round to divisible by 8
+            print(f"prompt: {prompt}")
+            print(f"negative_prompt: {negative_prompt}")
+            print(f"height: {height}")
+            print(f"width: {width}")
+            print(f"sample_steps: {sample_steps}")
+            print(f"scale: {scale}")
+            image = pipeline(
+                prompt=prompt,
+                height=height,
+                width=width,
+                num_inference_steps=sample_steps,
+                guidance_scale=scale,
+                negative_prompt=negative_prompt,
+                controlnet=controlnet,
+                controlnet_image=controlnet_image,
+            ).images[0]
+
+            ts_str = time.strftime("%Y%m%d%H%M%S", time.localtime())
+            num_suffix = f"e{epoch:06d}" if epoch is not None else f"{steps:06d}"
+            seed_suffix = "" if seed is None else f"_{seed}"
+            img_filename = (
+                f"{'' if args.output_name is None else args.output_name + '_'}{ts_str}_{num_suffix}_{i:02d}{seed_suffix}.png"
+            )
+
+            image.save(os.path.join(save_dir, img_filename))
+
+            # wandb有効時のみログを送信
+            try:
+                wandb_tracker = accelerator.get_tracker("wandb")
+                try:
+                    import wandb
+                except ImportError:  # 事前に一度確認するのでここはエラー出ないはず
+                    raise ImportError("No wandb / wandb がインストールされていないようです")
+
+                wandb_tracker.log({f"sample_{i}": wandb.Image(image)})
+            except:  # wandb 無効時
+                pass
+
+    # clear pipeline and cache to reduce vram usage
+    del pipeline
+    torch.cuda.empty_cache()
+
+    torch.set_rng_state(rng_state)
+    if cuda_rng_state is not None:
+        torch.cuda.set_rng_state(cuda_rng_state)
+    vae.to(org_vae_device)
+
+
+# endregion
+
+# region 前処理用
+
+
+class ImageLoadingDataset(torch.utils.data.Dataset):
+    def __init__(self, image_paths):
+        self.images = image_paths
+
+    def __len__(self):
+        return len(self.images)
+
+    def __getitem__(self, idx):
+        img_path = self.images[idx]
+
+        try:
+            image = Image.open(img_path).convert("RGB")
+            # convert to tensor temporarily so dataloader will accept it
+            tensor_pil = transforms.functional.pil_to_tensor(image)
+        except Exception as e:
+            print(f"Could not load image path / 画像を読み込めません: {img_path}, error: {e}")
+            return None
+
+        return (tensor_pil, img_path)
+
+
+# endregion
+
+
+# collate_fn用 epoch,stepはmultiprocessing.Value
+class collater_class:
+    def __init__(self, epoch, step, dataset):
+        self.current_epoch = epoch
+        self.current_step = step
+        self.dataset = dataset  # not used if worker_info is not None, in case of multiprocessing
+
+    def __call__(self, examples):
+        worker_info = torch.utils.data.get_worker_info()
+        # worker_info is None in the main process
+        if worker_info is not None:
+            dataset = worker_info.dataset
+        else:
+            dataset = self.dataset
+
+        # set epoch and step
+        dataset.set_current_epoch(self.current_epoch.value)
+        dataset.set_current_step(self.current_step.value)
+        return examples[0]
diff --git a/library/utilities.py b/library/utilities.py
new file mode 100644
index 0000000000000000000000000000000000000000..611bd06ab412f8d34a1661103a47aa8061a32657
--- /dev/null
+++ b/library/utilities.py
@@ -0,0 +1,96 @@
+# v1: initial release
+# v2: add open and save folder icons
+# v3: Add new Utilities tab for Dreambooth folder preparation
+# v3.1: Adding captionning of images to utilities
+
+import gradio as gr
+import os
+import argparse
+from library.basic_caption_gui import gradio_basic_caption_gui_tab
+from library.convert_model_gui import gradio_convert_model_tab
+from library.blip_caption_gui import gradio_blip_caption_gui_tab
+from library.git_caption_gui import gradio_git_caption_gui_tab
+from library.wd14_caption_gui import gradio_wd14_caption_gui_tab
+from library.group_images_gui import gradio_group_images_gui_tab
+
+
+def utilities_tab(
+    train_data_dir_input=gr.Textbox(),
+    reg_data_dir_input=gr.Textbox(),
+    output_dir_input=gr.Textbox(),
+    logging_dir_input=gr.Textbox(),
+    enable_copy_info_button=bool(False),
+    enable_dreambooth_tab=True,
+    headless=False
+):
+    with gr.Tab('Captioning'):
+        gradio_basic_caption_gui_tab(headless=headless)
+        gradio_blip_caption_gui_tab(headless=headless)
+        gradio_git_caption_gui_tab(headless=headless)
+        gradio_wd14_caption_gui_tab(headless=headless)
+    gradio_convert_model_tab(headless=headless)
+    gradio_group_images_gui_tab(headless=headless)
+
+    return (
+        train_data_dir_input,
+        reg_data_dir_input,
+        output_dir_input,
+        logging_dir_input,
+    )
+
+
+def UI(**kwargs):
+    css = ''
+
+    if os.path.exists('./style.css'):
+        with open(os.path.join('./style.css'), 'r', encoding='utf8') as file:
+            print('Load CSS...')
+            css += file.read() + '\n'
+
+    interface = gr.Blocks(css=css)
+
+    with interface:
+        utilities_tab()
+
+    # Show the interface
+    launch_kwargs = {}
+    if not kwargs.get('username', None) == '':
+        launch_kwargs['auth'] = (
+            kwargs.get('username', None),
+            kwargs.get('password', None),
+        )
+    if kwargs.get('server_port', 0) > 0:
+        launch_kwargs['server_port'] = kwargs.get('server_port', 0)
+    if kwargs.get('inbrowser', False):
+        launch_kwargs['inbrowser'] = kwargs.get('inbrowser', False)
+    print(launch_kwargs)
+    interface.launch(**launch_kwargs)
+
+
+if __name__ == '__main__':
+    # torch.cuda.set_per_process_memory_fraction(0.48)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--username', type=str, default='', help='Username for authentication'
+    )
+    parser.add_argument(
+        '--password', type=str, default='', help='Password for authentication'
+    )
+    parser.add_argument(
+        '--server_port',
+        type=int,
+        default=0,
+        help='Port to run the server listener on',
+    )
+    parser.add_argument(
+        '--inbrowser', action='store_true', help='Open in browser'
+    )
+
+    args = parser.parse_args()
+
+    UI(
+        username=args.username,
+        password=args.password,
+        inbrowser=args.inbrowser,
+        server_port=args.server_port,
+    )
diff --git a/library/utils.py b/library/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..7d801a676da99e7815957dbe6d25a878ae8dfcfc
--- /dev/null
+++ b/library/utils.py
@@ -0,0 +1,6 @@
+import threading
+from typing import *
+
+
+def fire_in_thread(f, *args, **kwargs):
+    threading.Thread(target=f, args=args, kwargs=kwargs).start()
\ No newline at end of file
diff --git a/library/verify_lora_gui.py b/library/verify_lora_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..b98abf66dc091eb1834f8d48eb188a2a06cc875f
--- /dev/null
+++ b/library/verify_lora_gui.py
@@ -0,0 +1,109 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+import os
+from .common_gui import (
+    get_saveasfilename_path,
+    get_any_file_path,
+    get_file_path,
+)
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe'
+folder_symbol = '\U0001f4c2'  # 📂
+refresh_symbol = '\U0001f504'  # 🔄
+save_style_symbol = '\U0001f4be'  # 💾
+document_symbol = '\U0001F4C4'   # 📄
+
+
+def verify_lora(
+    lora_model,
+):
+    # verify for caption_text_input
+    if lora_model == '':
+        msgbox('Invalid model A file')
+        return
+
+    # verify if source model exist
+    if not os.path.isfile(lora_model):
+        msgbox('The provided model A is not a file')
+        return
+
+    run_cmd = [
+        PYTHON,
+        os.path.join('networks', 'check_lora_weights.py'),
+        f'{lora_model}',
+    ]
+
+    log.info(' '.join(run_cmd))
+
+    # Run the command
+    process = subprocess.Popen(
+        run_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
+    output, error = process.communicate()
+
+    return (output.decode(), error.decode())
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_verify_lora_tab(headless=False):
+    with gr.Tab('Verify LoRA'):
+        gr.Markdown(
+            'This utility can verify a LoRA network to make sure it is properly trained.'
+        )
+
+        lora_ext = gr.Textbox(value='*.pt *.safetensors', visible=False)
+        lora_ext_name = gr.Textbox(value='LoRA model types', visible=False)
+
+        with gr.Row():
+            lora_model = gr.Textbox(
+                label='LoRA model',
+                placeholder='Path to the LoRA model to verify',
+                interactive=True,
+            )
+            button_lora_model_file = gr.Button(
+                folder_symbol,
+                elem_id='open_folder_small',
+                visible=(not headless),
+            )
+            button_lora_model_file.click(
+                get_file_path,
+                inputs=[lora_model, lora_ext, lora_ext_name],
+                outputs=lora_model,
+                show_progress=False,
+            )
+            verify_button = gr.Button('Verify', variant='primary')
+
+        lora_model_verif_output = gr.Textbox(
+            label='Output',
+            placeholder='Verification output',
+            interactive=False,
+            lines=1,
+            max_lines=10,
+        )
+
+        lora_model_verif_error = gr.Textbox(
+            label='Error',
+            placeholder='Verification error',
+            interactive=False,
+            lines=1,
+            max_lines=10,
+        )
+
+        verify_button.click(
+            verify_lora,
+            inputs=[
+                lora_model,
+            ],
+            outputs=[lora_model_verif_output, lora_model_verif_error],
+            show_progress=False,
+        )
diff --git a/library/wd14_caption_gui.py b/library/wd14_caption_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..dae3909b26edd749632913485407331507648843
--- /dev/null
+++ b/library/wd14_caption_gui.py
@@ -0,0 +1,219 @@
+import gradio as gr
+from easygui import msgbox
+import subprocess
+from .common_gui import get_folder_path, add_pre_postfix
+import os
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+
+def caption_images(
+    train_data_dir,
+    caption_extension,
+    batch_size,
+    general_threshold,
+    character_threshold,
+    replace_underscores,
+    model,
+    recursive,
+    max_data_loader_n_workers,
+    debug,
+    undesired_tags,
+    frequency_tags,
+    prefix,
+    postfix,
+):
+    # Check for images_dir_input
+    if train_data_dir == '':
+        msgbox('Image folder is missing...')
+        return
+
+    if caption_extension == '':
+        msgbox('Please provide an extension for the caption files.')
+        return
+
+    log.info(f'Captioning files in {train_data_dir}...')
+    run_cmd = f'accelerate launch "./finetune/tag_images_by_wd14_tagger.py"'
+    run_cmd += f' --batch_size={int(batch_size)}'
+    run_cmd += f' --general_threshold={general_threshold}'
+    run_cmd += f' --character_threshold={character_threshold}'
+    run_cmd += f' --caption_extension="{caption_extension}"'
+    run_cmd += f' --model="{model}"'
+    run_cmd += (
+        f' --max_data_loader_n_workers="{int(max_data_loader_n_workers)}"'
+    )
+
+    if recursive:
+        run_cmd += f' --recursive'
+    if debug:
+        run_cmd += f' --debug'
+    if replace_underscores:
+        run_cmd += f' --remove_underscore'
+    if frequency_tags:
+        run_cmd += f' --frequency_tags'
+
+    if not undesired_tags == '':
+        run_cmd += f' --undesired_tags="{undesired_tags}"'
+    run_cmd += f' "{train_data_dir}"'
+
+    log.info(run_cmd)
+
+    # Run the command
+    if os.name == 'posix':
+        os.system(run_cmd)
+    else:
+        subprocess.run(run_cmd)
+
+    # Add prefix and postfix
+    add_pre_postfix(
+        folder=train_data_dir,
+        caption_file_ext=caption_extension,
+        prefix=prefix,
+        postfix=postfix,
+    )
+
+    log.info('...captioning done')
+
+
+###
+# Gradio UI
+###
+
+
+def gradio_wd14_caption_gui_tab(headless=False):
+    with gr.Tab('WD14 Captioning'):
+        gr.Markdown(
+            'This utility will use WD14 to caption files for each images in a folder.'
+        )
+
+        # Input Settings
+        # with gr.Section('Input Settings'):
+        with gr.Row():
+            train_data_dir = gr.Textbox(
+                label='Image folder to caption',
+                placeholder='Directory containing the images to caption',
+                interactive=True,
+            )
+            button_train_data_dir_input = gr.Button(
+                '📂', elem_id='open_folder_small', visible=(not headless)
+            )
+            button_train_data_dir_input.click(
+                get_folder_path,
+                outputs=train_data_dir,
+                show_progress=False,
+            )
+
+            caption_extension = gr.Textbox(
+                label='Caption file extension',
+                placeholder='Extention for caption file. eg: .caption, .txt',
+                value='.txt',
+                interactive=True,
+            )
+
+        undesired_tags = gr.Textbox(
+            label='Undesired tags',
+            placeholder='(Optional) Separate `undesired_tags` with comma `(,)` if you want to remove multiple tags, e.g. `1girl,solo,smile`.',
+            interactive=True,
+        )
+
+        with gr.Row():
+            prefix = gr.Textbox(
+                label='Prefix to add to WD14 caption',
+                placeholder='(Optional)',
+                interactive=True,
+            )
+
+            postfix = gr.Textbox(
+                label='Postfix to add to WD14 caption',
+                placeholder='(Optional)',
+                interactive=True,
+            )
+
+        with gr.Row():
+            replace_underscores = gr.Checkbox(
+                label='Replace underscores in filenames with spaces',
+                value=True,
+                interactive=True,
+            )
+            recursive = gr.Checkbox(
+                label='Recursive',
+                value=False,
+                info='Tag subfolders images as well',
+            )
+
+            debug = gr.Checkbox(
+                label='Verbose logging',
+                value=True,
+                info='Debug while tagging, it will print your image file with general tags and character tags.',
+            )
+            frequency_tags = gr.Checkbox(
+                label='Show tags frequency',
+                value=True,
+                info='Show frequency of tags for images.',
+            )
+
+        # Model Settings
+        with gr.Row():
+            model = gr.Dropdown(
+                label='Model',
+                choices=[
+                    'SmilingWolf/wd-v1-4-convnext-tagger-v2',
+                    'SmilingWolf/wd-v1-4-convnextv2-tagger-v2',
+                    'SmilingWolf/wd-v1-4-vit-tagger-v2',
+                    'SmilingWolf/wd-v1-4-swinv2-tagger-v2',
+                ],
+                value='SmilingWolf/wd-v1-4-convnextv2-tagger-v2',
+            )
+
+            general_threshold = gr.Slider(
+                value=0.35,
+                label='General threshold',
+                info='Adjust `general_threshold` for pruning tags (less tags, less flexible)',
+                minimum=0,
+                maximum=1,
+                step=0.05,
+            )
+            character_threshold = gr.Slider(
+                value=0.35,
+                label='Character threshold',
+                info='useful if you want to train with character',
+                minimum=0,
+                maximum=1,
+                step=0.05,
+            )
+
+        # Advanced Settings
+        with gr.Row():
+            batch_size = gr.Number(
+                value=8, label='Batch size', interactive=True
+            )
+
+            max_data_loader_n_workers = gr.Number(
+                value=2, label='Max dataloader workers', interactive=True
+            )
+
+        caption_button = gr.Button('Caption images')
+
+        caption_button.click(
+            caption_images,
+            inputs=[
+                train_data_dir,
+                caption_extension,
+                batch_size,
+                general_threshold,
+                character_threshold,
+                replace_underscores,
+                model,
+                recursive,
+                max_data_loader_n_workers,
+                debug,
+                undesired_tags,
+                frequency_tags,
+                prefix,
+                postfix,
+            ],
+            show_progress=False,
+        )
diff --git a/lora_gui.py b/lora_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a6fc49d01a70aec9e7e0bc324aec48ba8936ce8
--- /dev/null
+++ b/lora_gui.py
@@ -0,0 +1,1687 @@
+# v1: initial release
+# v2: add open and save folder icons
+# v3: Add new Utilities tab for Dreambooth folder preparation
+# v3.1: Adding captionning of images to utilities
+
+import gradio as gr
+import json
+import math
+import os
+import subprocess
+import pathlib
+import argparse
+from datetime import datetime
+from library.common_gui import (
+    get_file_path,
+    get_any_file_path,
+    get_saveasfile_path,
+    color_aug_changed,
+    save_inference_file,
+    run_cmd_advanced_training,
+    run_cmd_training,
+    update_my_data,
+    check_if_model_exist,
+    output_message,
+    verify_image_folder_pattern,
+    SaveConfigFile,
+    save_to_file
+)
+from library.class_configuration_file import ConfigurationFile
+from library.class_source_model import SourceModel
+from library.class_basic_training import BasicTraining
+from library.class_advanced_training import AdvancedTraining
+from library.class_sdxl_parameters import SDXLParameters
+from library.class_folders import Folders
+from library.tensorboard_gui import (
+    gradio_tensorboard,
+    start_tensorboard,
+    stop_tensorboard,
+)
+from library.utilities import utilities_tab
+from library.class_sample_images import SampleImages, run_cmd_sample
+from library.class_lora_tab import LoRATools
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+document_symbol = '\U0001F4C4'   # 📄
+
+
+def save_configuration(
+    save_as,
+    file_path,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl,
+    logging_dir,
+    train_data_dir,
+    reg_data_dir,
+    output_dir,
+    max_resolution,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    cache_latents,
+    cache_latents_to_disk,
+    caption_extension,
+    enable_bucket,
+    gradient_checkpointing,
+    full_fp16,
+    no_token_padding,
+    stop_text_encoder_training,
+    # use_8bit_adam,
+    xformers,
+    save_model_as,
+    shuffle_caption,
+    save_state,
+    resume,
+    prior_loss_weight,
+    text_encoder_lr,
+    unet_lr,
+    network_dim,
+    lora_network_weights,
+    dim_from_weights,
+    color_aug,
+    flip_aug,
+    clip_skip,
+    gradient_accumulation_steps,
+    mem_eff_attn,
+    output_name,
+    model_list,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    network_alpha,
+    training_comment,
+    keep_tokens,
+    lr_scheduler_num_cycles,
+    lr_scheduler_power,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    LoRA_type,
+    factor,
+    use_cp,
+    decompose_both,
+    train_on_input,
+    conv_dim,
+    conv_alpha,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    down_lr_weight,
+    mid_lr_weight,
+    up_lr_weight,
+    block_lr_zero_threshold,
+    block_dims,
+    block_alphas,
+    conv_dims,
+    conv_alphas,
+    weighted_captions,
+    unit,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    scale_weight_norms,
+    network_dropout,
+    rank_dropout,
+    module_dropout,
+    sdxl_cache_text_encoder_outputs,
+    sdxl_no_half_vae,
+    min_timestep,
+    max_timestep,
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+
+    original_file_path = file_path
+
+    save_as_bool = True if save_as.get('label') == 'True' else False
+
+    if save_as_bool:
+        log.info('Save as...')
+        file_path = get_saveasfile_path(file_path)
+    else:
+        log.info('Save...')
+        if file_path == None or file_path == '':
+            file_path = get_saveasfile_path(file_path)
+
+    # log.info(file_path)
+
+    if file_path == None or file_path == '':
+        return original_file_path  # In case a file_path was provided and the user decide to cancel the open action
+
+    # Extract the destination directory from the file path
+    destination_directory = os.path.dirname(file_path)
+
+    # Create the destination directory if it doesn't exist
+    if not os.path.exists(destination_directory):
+        os.makedirs(destination_directory)
+
+    SaveConfigFile(parameters=parameters, file_path=file_path, exclusion=['file_path', 'save_as'])
+
+    return file_path
+
+
+def open_configuration(
+    ask_for_file,
+    apply_preset,
+    file_path,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl,
+    logging_dir,
+    train_data_dir,
+    reg_data_dir,
+    output_dir,
+    max_resolution,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    cache_latents,
+    cache_latents_to_disk,
+    caption_extension,
+    enable_bucket,
+    gradient_checkpointing,
+    full_fp16,
+    no_token_padding,
+    stop_text_encoder_training,
+    # use_8bit_adam,
+    xformers,
+    save_model_as,
+    shuffle_caption,
+    save_state,
+    resume,
+    prior_loss_weight,
+    text_encoder_lr,
+    unet_lr,
+    network_dim,
+    lora_network_weights,
+    dim_from_weights,
+    color_aug,
+    flip_aug,
+    clip_skip,
+    gradient_accumulation_steps,
+    mem_eff_attn,
+    output_name,
+    model_list,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    network_alpha,
+    training_comment,
+    keep_tokens,
+    lr_scheduler_num_cycles,
+    lr_scheduler_power,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    LoRA_type,
+    factor,
+    use_cp,
+    decompose_both,
+    train_on_input,
+    conv_dim,
+    conv_alpha,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    down_lr_weight,
+    mid_lr_weight,
+    up_lr_weight,
+    block_lr_zero_threshold,
+    block_dims,
+    block_alphas,
+    conv_dims,
+    conv_alphas,
+    weighted_captions,
+    unit,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    scale_weight_norms,
+    network_dropout,
+    rank_dropout,
+    module_dropout,
+    sdxl_cache_text_encoder_outputs,
+    sdxl_no_half_vae,
+    min_timestep,
+    max_timestep,
+    training_preset,
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+
+    ask_for_file = True if ask_for_file.get('label') == 'True' else False
+    apply_preset = True if apply_preset.get('label') == 'True' else False
+
+    # Check if we are "applying" a preset or a config
+    if apply_preset:
+        log.info(f'Applying preset {training_preset}...')
+        file_path = f'./presets/lora/{training_preset}.json'
+    else:
+        # If not applying a preset, set the `training_preset` field to an empty string
+        # Find the index of the `training_preset` parameter using the `index()` method
+        training_preset_index = parameters.index(
+            ('training_preset', training_preset)
+        )
+
+        # Update the value of `training_preset` by directly assigning an empty string value
+        parameters[training_preset_index] = ('training_preset', '')
+
+    original_file_path = file_path
+
+    if ask_for_file:
+        file_path = get_file_path(file_path)
+
+    if not file_path == '' and not file_path == None:
+        # Load variables from JSON file
+        with open(file_path, 'r') as f:
+            my_data = json.load(f)
+            log.info('Loading config...')
+
+            # Update values to fix deprecated options, set appropriate optimizer if it is set to True, etc.
+            my_data = update_my_data(my_data)
+    else:
+        file_path = original_file_path  # In case a file_path was provided and the user decides to cancel the open action
+        my_data = {}
+
+    values = [file_path]
+    for key, value in parameters:
+        # Set the value in the dictionary to the corresponding value in `my_data`, or the default value if not found
+        if not key in ['ask_for_file', 'apply_preset', 'file_path']:
+            json_value = my_data.get(key)
+            # if isinstance(json_value, str) and json_value == '':
+            #     # If the JSON value is an empty string, use the default value
+            #     values.append(value)
+            # else:
+            # Otherwise, use the JSON value if not None, otherwise use the default value
+            values.append(json_value if json_value is not None else value)
+
+    # This next section is about making the LoCon parameters visible if LoRA_type = 'Standard'
+    if my_data.get('LoRA_type', 'Standard') == 'LoCon':
+        values.append(gr.Row.update(visible=True))
+    else:
+        values.append(gr.Row.update(visible=False))
+
+    return tuple(values)
+
+
+def train_model(
+    headless,
+    print_only,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl,
+    logging_dir,
+    train_data_dir,
+    reg_data_dir,
+    output_dir,
+    max_resolution,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    cache_latents,
+    cache_latents_to_disk,
+    caption_extension,
+    enable_bucket,
+    gradient_checkpointing,
+    full_fp16,
+    no_token_padding,
+    stop_text_encoder_training_pct,
+    # use_8bit_adam,
+    xformers,
+    save_model_as,
+    shuffle_caption,
+    save_state,
+    resume,
+    prior_loss_weight,
+    text_encoder_lr,
+    unet_lr,
+    network_dim,
+    lora_network_weights,
+    dim_from_weights,
+    color_aug,
+    flip_aug,
+    clip_skip,
+    gradient_accumulation_steps,
+    mem_eff_attn,
+    output_name,
+    model_list,  # Keep this. Yes, it is unused here but required given the common list used
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    network_alpha,
+    training_comment,
+    keep_tokens,
+    lr_scheduler_num_cycles,
+    lr_scheduler_power,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    LoRA_type,
+    factor,
+    use_cp,
+    decompose_both,
+    train_on_input,
+    conv_dim,
+    conv_alpha,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    down_lr_weight,
+    mid_lr_weight,
+    up_lr_weight,
+    block_lr_zero_threshold,
+    block_dims,
+    block_alphas,
+    conv_dims,
+    conv_alphas,
+    weighted_captions,
+    unit,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    scale_weight_norms,
+    network_dropout,
+    rank_dropout,
+    module_dropout,
+    sdxl_cache_text_encoder_outputs,
+    sdxl_no_half_vae,
+    min_timestep,
+    max_timestep,
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+    
+    print_only_bool = True if print_only.get('label') == 'True' else False
+    log.info(f'Start training LoRA {LoRA_type} ...')
+    headless_bool = True if headless.get('label') == 'True' else False
+
+    if pretrained_model_name_or_path == '':
+        output_message(
+            msg='Source model information is missing', headless=headless_bool
+        )
+        return
+
+    if train_data_dir == '':
+        output_message(
+            msg='Image folder path is missing', headless=headless_bool
+        )
+        return
+
+    if not os.path.exists(train_data_dir):
+        output_message(
+            msg='Image folder does not exist', headless=headless_bool
+        )
+        return
+
+    if not verify_image_folder_pattern(train_data_dir):
+        return
+
+    if reg_data_dir != '':
+        if not os.path.exists(reg_data_dir):
+            output_message(
+                msg='Regularisation folder does not exist',
+                headless=headless_bool,
+            )
+            return
+
+        if not verify_image_folder_pattern(reg_data_dir):
+            return
+
+    if output_dir == '':
+        output_message(
+            msg='Output folder path is missing', headless=headless_bool
+        )
+        return
+
+    if int(bucket_reso_steps) < 1:
+        output_message(
+            msg='Bucket resolution steps need to be greater than 0',
+            headless=headless_bool,
+        )
+        return
+
+    if noise_offset == '':
+        noise_offset = 0
+
+    if float(noise_offset) > 1 or float(noise_offset) < 0:
+        output_message(
+            msg='Noise offset need to be a value between 0 and 1',
+            headless=headless_bool,
+        )
+        return
+
+    # if float(noise_offset) > 0 and (
+    #     multires_noise_iterations > 0 or multires_noise_discount > 0
+    # ):
+    #     output_message(
+    #         msg="noise offset and multires_noise can't be set at the same time. Only use one or the other.",
+    #         title='Error',
+    #         headless=headless_bool,
+    #     )
+    #     return
+
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    if stop_text_encoder_training_pct > 0:
+        output_message(
+            msg='Output "stop text encoder training" is not yet supported. Ignoring',
+            headless=headless_bool,
+        )
+        stop_text_encoder_training_pct = 0
+
+    if check_if_model_exist(
+        output_name, output_dir, save_model_as, headless=headless_bool
+    ):
+        return
+
+    # if optimizer == 'Adafactor' and lr_warmup != '0':
+    #     output_message(
+    #         msg="Warning: lr_scheduler is set to 'Adafactor', so 'LR warmup (% of steps)' will be considered 0.",
+    #         title='Warning',
+    #         headless=headless_bool,
+    #     )
+    #     lr_warmup = '0'
+
+    # If string is empty set string to 0.
+    if text_encoder_lr == '':
+        text_encoder_lr = 0
+    if unet_lr == '':
+        unet_lr = 0
+
+    # Get a list of all subfolders in train_data_dir
+    subfolders = [
+        f
+        for f in os.listdir(train_data_dir)
+        if os.path.isdir(os.path.join(train_data_dir, f))
+    ]
+
+    total_steps = 0
+
+    # Loop through each subfolder and extract the number of repeats
+    for folder in subfolders:
+        try:
+            # Extract the number of repeats from the folder name
+            repeats = int(folder.split('_')[0])
+
+            # Count the number of images in the folder
+            num_images = len(
+                [
+                    f
+                    for f, lower_f in (
+                        (file, file.lower())
+                        for file in os.listdir(
+                            os.path.join(train_data_dir, folder)
+                        )
+                    )
+                    if lower_f.endswith(('.jpg', '.jpeg', '.png', '.webp'))
+                ]
+            )
+
+            log.info(f'Folder {folder}: {num_images} images found')
+
+            # Calculate the total number of steps for this folder
+            steps = repeats * num_images
+
+            # log.info the result
+            log.info(f'Folder {folder}: {steps} steps')
+
+            total_steps += steps
+
+        except ValueError:
+            # Handle the case where the folder name does not contain an underscore
+            log.info(
+                f"Error: '{folder}' does not contain an underscore, skipping..."
+            )
+
+    if reg_data_dir == '':
+        reg_factor = 1
+    else:
+        log.info(
+            '\033[94mRegularisation images are used... Will double the number of steps required...\033[0m'
+        )
+        reg_factor = 2
+
+    log.info(f'Total steps: {total_steps}')
+    log.info(f'Train batch size: {train_batch_size}')
+    log.info(f'Gradient accumulation steps: {gradient_accumulation_steps}')
+    log.info(f'Epoch: {epoch}')
+    log.info(f'Regulatization factor: {reg_factor}')
+
+    # calculate max_train_steps
+    max_train_steps = int(
+        math.ceil(
+            float(total_steps)
+            / int(train_batch_size)
+            / int(gradient_accumulation_steps)
+            * int(epoch)
+            * int(reg_factor)
+        )
+    )
+    log.info(
+        f'max_train_steps ({total_steps} / {train_batch_size} / {gradient_accumulation_steps} * {epoch} * {reg_factor}) = {max_train_steps}'
+    )
+
+    # calculate stop encoder training
+    if stop_text_encoder_training_pct == None:
+        stop_text_encoder_training = 0
+    else:
+        stop_text_encoder_training = math.ceil(
+            float(max_train_steps) / 100 * int(stop_text_encoder_training_pct)
+        )
+    log.info(f'stop_text_encoder_training = {stop_text_encoder_training}')
+
+    lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+    log.info(f'lr_warmup_steps = {lr_warmup_steps}')
+
+    run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process}'
+    if sdxl:
+        run_cmd += f' "./sdxl_train_network.py"'
+    else:
+        run_cmd += f' "./train_network.py"'
+
+    if v2:
+        run_cmd += ' --v2'
+    if v_parameterization:
+        run_cmd += ' --v_parameterization'
+    if enable_bucket:
+        run_cmd += ' --enable_bucket'
+    if no_token_padding:
+        run_cmd += ' --no_token_padding'
+    if weighted_captions:
+        run_cmd += ' --weighted_captions'
+    run_cmd += (
+        f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"'
+    )
+    run_cmd += f' --train_data_dir="{train_data_dir}"'
+    if len(reg_data_dir):
+        run_cmd += f' --reg_data_dir="{reg_data_dir}"'
+    run_cmd += f' --resolution="{max_resolution}"'
+    run_cmd += f' --output_dir="{output_dir}"'
+    if not logging_dir == '':
+        run_cmd += f' --logging_dir="{logging_dir}"'
+    run_cmd += f' --network_alpha="{network_alpha}"'
+    if not training_comment == '':
+        run_cmd += f' --training_comment="{training_comment}"'
+    if not stop_text_encoder_training == 0:
+        run_cmd += (
+            f' --stop_text_encoder_training={stop_text_encoder_training}'
+        )
+    if not save_model_as == 'same as source model':
+        run_cmd += f' --save_model_as={save_model_as}'
+    if not float(prior_loss_weight) == 1.0:
+        run_cmd += f' --prior_loss_weight={prior_loss_weight}'
+
+    if LoRA_type == 'LoCon' or LoRA_type == 'LyCORIS/LoCon':
+        try:
+            import lycoris
+        except ModuleNotFoundError:
+            log.info(
+                "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
+            )
+            return
+        run_cmd += f' --network_module=lycoris.kohya'
+        run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=lora"'
+
+    if LoRA_type == 'LyCORIS/LoHa':
+        try:
+            import lycoris
+        except ModuleNotFoundError:
+            log.info(
+                "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
+            )
+            return
+        run_cmd += f' --network_module=lycoris.kohya'
+        run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "algo=loha"'
+        # This is a hack to fix a train_network LoHA logic issue
+        if not network_dropout > 0.0:
+            run_cmd += f' --network_dropout="{network_dropout}"'
+
+    if LoRA_type == 'LyCORIS/iA3':
+        try:
+            import lycoris
+        except ModuleNotFoundError:
+            log.info(
+                "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
+            )
+            return
+        run_cmd += f' --network_module=lycoris.kohya'
+        run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "train_on_input={train_on_input}" "algo=ia3"'
+        # This is a hack to fix a train_network LoHA logic issue
+        if not network_dropout > 0.0:
+            run_cmd += f' --network_dropout="{network_dropout}"'
+
+    if LoRA_type == 'LyCORIS/DyLoRA':
+        try:
+            import lycoris
+        except ModuleNotFoundError:
+            log.info(
+                "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
+            )
+            return
+        run_cmd += f' --network_module=lycoris.kohya'
+        run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "block_size={unit}" "algo=dylora"'
+        # This is a hack to fix a train_network LoHA logic issue
+        if not network_dropout > 0.0:
+            run_cmd += f' --network_dropout="{network_dropout}"'
+
+    if LoRA_type == 'LyCORIS/LoKr':
+        try:
+            import lycoris
+        except ModuleNotFoundError:
+            log.info(
+                "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
+            )
+            return
+        run_cmd += f' --network_module=lycoris.kohya'
+        run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "factor={factor}" "use_cp={use_cp}" "algo=lokr"'
+        # This is a hack to fix a train_network LoHA logic issue
+        if not network_dropout > 0.0:
+            run_cmd += f' --network_dropout="{network_dropout}"'
+
+    if LoRA_type in ['Kohya LoCon', 'Standard']:
+        kohya_lora_var_list = [
+            'down_lr_weight',
+            'mid_lr_weight',
+            'up_lr_weight',
+            'block_lr_zero_threshold',
+            'block_dims',
+            'block_alphas',
+            'conv_dims',
+            'conv_alphas',
+            'rank_dropout',
+            'module_dropout',
+        ]
+
+        run_cmd += f' --network_module=networks.lora'
+        kohya_lora_vars = {
+            key: value
+            for key, value in vars().items()
+            if key in kohya_lora_var_list and value
+        }
+
+        network_args = ''
+        if LoRA_type == 'Kohya LoCon':
+            network_args += f' conv_dim="{conv_dim}" conv_alpha="{conv_alpha}"'
+
+        for key, value in kohya_lora_vars.items():
+            if value:
+                network_args += f' {key}="{value}"'
+
+        if network_args:
+            run_cmd += f' --network_args{network_args}'
+
+    if LoRA_type in ['Kohya DyLoRA']:
+        kohya_lora_var_list = [
+            'conv_dim',
+            'conv_alpha',
+            'down_lr_weight',
+            'mid_lr_weight',
+            'up_lr_weight',
+            'block_lr_zero_threshold',
+            'block_dims',
+            'block_alphas',
+            'conv_dims',
+            'conv_alphas',
+            'rank_dropout',
+            'module_dropout',
+            'unit',
+        ]
+
+        run_cmd += f' --network_module=networks.dylora'
+        kohya_lora_vars = {
+            key: value
+            for key, value in vars().items()
+            if key in kohya_lora_var_list and value
+        }
+
+        network_args = ''
+
+        for key, value in kohya_lora_vars.items():
+            if value:
+                network_args += f' {key}="{value}"'
+
+        if network_args:
+            run_cmd += f' --network_args{network_args}'
+
+    if not (float(text_encoder_lr) == 0) or not (float(unet_lr) == 0):
+        if not (float(text_encoder_lr) == 0) and not (float(unet_lr) == 0):
+            run_cmd += f' --text_encoder_lr={text_encoder_lr}'
+            run_cmd += f' --unet_lr={unet_lr}'
+        elif not (float(text_encoder_lr) == 0):
+            run_cmd += f' --text_encoder_lr={text_encoder_lr}'
+            run_cmd += f' --network_train_text_encoder_only'
+        else:
+            run_cmd += f' --unet_lr={unet_lr}'
+            run_cmd += f' --network_train_unet_only'
+    else:
+        if float(learning_rate) == 0:
+            output_message(
+                msg='Please input learning rate values.',
+                headless=headless_bool,
+            )
+            return
+
+    run_cmd += f' --network_dim={network_dim}'
+
+    #if LoRA_type not in ['LyCORIS/LoCon']:
+    if not lora_network_weights == '':
+        run_cmd += f' --network_weights="{lora_network_weights}"'
+        if dim_from_weights:
+            run_cmd += f' --dim_from_weights'
+
+    if int(gradient_accumulation_steps) > 1:
+        run_cmd += f' --gradient_accumulation_steps={int(gradient_accumulation_steps)}'
+    if not output_name == '':
+        run_cmd += f' --output_name="{output_name}"'
+    if not lr_scheduler_num_cycles == '':
+        run_cmd += f' --lr_scheduler_num_cycles="{lr_scheduler_num_cycles}"'
+    else:
+        run_cmd += f' --lr_scheduler_num_cycles="{epoch}"'
+    if not lr_scheduler_power == '':
+        run_cmd += f' --lr_scheduler_power="{lr_scheduler_power}"'
+
+    if scale_weight_norms > 0.0:
+        run_cmd += f' --scale_weight_norms="{scale_weight_norms}"'
+
+    if network_dropout > 0.0:
+        run_cmd += f' --network_dropout="{network_dropout}"'
+        
+    if sdxl_cache_text_encoder_outputs:
+        run_cmd += f' --cache_text_encoder_outputs'
+        
+    if sdxl_no_half_vae:
+        run_cmd += f' --no_half_vae'
+
+    run_cmd += run_cmd_training(
+        learning_rate=learning_rate,
+        lr_scheduler=lr_scheduler,
+        lr_warmup_steps=lr_warmup_steps,
+        train_batch_size=train_batch_size,
+        max_train_steps=max_train_steps,
+        save_every_n_epochs=save_every_n_epochs,
+        mixed_precision=mixed_precision,
+        save_precision=save_precision,
+        seed=seed,
+        caption_extension=caption_extension,
+        cache_latents=cache_latents,
+        cache_latents_to_disk=cache_latents_to_disk,
+        optimizer=optimizer,
+        optimizer_args=optimizer_args,
+    )
+
+    run_cmd += run_cmd_advanced_training(
+        max_train_epochs=max_train_epochs,
+        max_data_loader_n_workers=max_data_loader_n_workers,
+        max_token_length=max_token_length,
+        resume=resume,
+        save_state=save_state,
+        mem_eff_attn=mem_eff_attn,
+        clip_skip=clip_skip,
+        flip_aug=flip_aug,
+        color_aug=color_aug,
+        shuffle_caption=shuffle_caption,
+        gradient_checkpointing=gradient_checkpointing,
+        full_fp16=full_fp16,
+        xformers=xformers,
+        # use_8bit_adam=use_8bit_adam,
+        keep_tokens=keep_tokens,
+        persistent_data_loader_workers=persistent_data_loader_workers,
+        bucket_no_upscale=bucket_no_upscale,
+        random_crop=random_crop,
+        bucket_reso_steps=bucket_reso_steps,
+        caption_dropout_every_n_epochs=caption_dropout_every_n_epochs,
+        caption_dropout_rate=caption_dropout_rate,
+        noise_offset_type=noise_offset_type,
+        noise_offset=noise_offset,
+        adaptive_noise_scale=adaptive_noise_scale,
+        multires_noise_iterations=multires_noise_iterations,
+        multires_noise_discount=multires_noise_discount,
+        additional_parameters=additional_parameters,
+        vae_batch_size=vae_batch_size,
+        min_snr_gamma=min_snr_gamma,
+        save_every_n_steps=save_every_n_steps,
+        save_last_n_steps=save_last_n_steps,
+        save_last_n_steps_state=save_last_n_steps_state,
+        use_wandb=use_wandb,
+        wandb_api_key=wandb_api_key,
+        scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred,
+        min_timestep=min_timestep,
+        max_timestep=max_timestep,
+    )
+
+    run_cmd += run_cmd_sample(
+        sample_every_n_steps,
+        sample_every_n_epochs,
+        sample_sampler,
+        sample_prompts,
+        output_dir,
+    )
+
+    if print_only_bool:
+        log.warning(
+            'Here is the trainer command as a reference. It will not be executed:\n'
+        )
+        print(run_cmd)
+        
+        save_to_file(run_cmd)
+    else:
+        # Saving config file for model
+        current_datetime = datetime.now()
+        formatted_datetime = current_datetime.strftime("%Y%m%d-%H%M%S")
+        file_path = os.path.join(output_dir, f'{output_name}_{formatted_datetime}.json')
+        
+        log.info(f'Saving training config to {file_path}...')
+
+        SaveConfigFile(parameters=parameters, file_path=file_path, exclusion=['file_path', 'save_as', 'headless', 'print_only'])
+        
+        log.info(run_cmd)
+        # Run the command
+        if os.name == 'posix':
+            os.system(run_cmd)
+        else:
+            subprocess.run(run_cmd)
+
+        # check if output_dir/last is a folder... therefore it is a diffuser model
+        last_dir = pathlib.Path(f'{output_dir}/{output_name}')
+
+        if not last_dir.is_dir():
+            # Copy inference model for v2 if required
+            save_inference_file(
+                output_dir, v2, v_parameterization, output_name
+            )
+
+
+def lora_tab(
+    train_data_dir_input=gr.Textbox(),
+    reg_data_dir_input=gr.Textbox(),
+    output_dir_input=gr.Textbox(),
+    logging_dir_input=gr.Textbox(),
+    headless=False,
+):
+    dummy_db_true = gr.Label(value=True, visible=False)
+    dummy_db_false = gr.Label(value=False, visible=False)
+    dummy_headless = gr.Label(value=headless, visible=False)
+
+    with gr.Tab('Training'):
+        gr.Markdown(
+            'Train a custom model using kohya train network LoRA python code...'
+        )
+        
+        # Setup Configuration Files Gradio
+        config = ConfigurationFile(headless)
+
+        source_model = SourceModel(
+            save_model_as_choices=[
+                'ckpt',
+                'safetensors',
+            ],
+            headless=headless,
+        )
+
+        with gr.Tab('Folders'):
+            folders = Folders(headless=headless)
+            
+        with gr.Tab('Parameters'):
+
+            def list_presets(path):
+                json_files = []
+                
+                for file in os.listdir(path):
+                    if file.endswith('.json'):
+                        json_files.append(os.path.splitext(file)[0])
+                        
+                user_presets_path = os.path.join(path, 'user_presets')
+                if os.path.isdir(user_presets_path):
+                    for file in os.listdir(user_presets_path):
+                        if file.endswith('.json'):
+                            preset_name = os.path.splitext(file)[0]
+                            json_files.append(os.path.join('user_presets', preset_name))
+                
+                return json_files
+            
+            training_preset = gr.Dropdown(
+                label='Presets',
+                choices=list_presets('./presets/lora'),
+                elem_id='myDropdown',
+            )
+            with gr.Row():
+                LoRA_type = gr.Dropdown(
+                    label='LoRA type',
+                    choices=[
+                        'Kohya DyLoRA',
+                        'Kohya LoCon',
+                        'LyCORIS/DyLoRA',
+                        'LyCORIS/iA3',
+                        'LyCORIS/LoCon',
+                        'LyCORIS/LoHa',
+                        'LyCORIS/LoKr',
+                        'Standard',
+                    ],
+                    value='Standard',
+                )
+                with gr.Box():
+                    with gr.Row():
+                        lora_network_weights = gr.Textbox(
+                            label='LoRA network weights',
+                            placeholder='(Optional)',
+                            info='Path to an existing LoRA network weights to resume training from',
+                        )
+                        lora_network_weights_file = gr.Button(
+                            document_symbol,
+                            elem_id='open_folder_small',
+                            visible=(not headless),
+                        )
+                        lora_network_weights_file.click(
+                            get_any_file_path,
+                            inputs=[lora_network_weights],
+                            outputs=lora_network_weights,
+                            show_progress=False,
+                        )
+                        dim_from_weights = gr.Checkbox(
+                            label='DIM from weights',
+                            value=False,
+                            info='Automatically determine the dim(rank) from the weight file.',
+                        )
+            basic_training = BasicTraining(
+                learning_rate_value='0.0001',
+                lr_scheduler_value='cosine',
+                lr_warmup_value='10',
+            )
+
+            with gr.Row():
+                text_encoder_lr = gr.Number(
+                    label='Text Encoder learning rate',
+                    value='5e-5',
+                    info='Optional',
+                )
+                unet_lr = gr.Number(
+                    label='Unet learning rate',
+                    value='0.0001',
+                    info='Optional',
+                )
+                
+            # Add SDXL Parameters
+            sdxl_params = SDXLParameters(source_model.sdxl_checkbox)
+                
+            with gr.Row():
+                factor = gr.Slider(
+                    label='LoKr factor',
+                    value=-1,
+                    minimum=-1,
+                    maximum=64,
+                    step=1,
+                    visible=False,
+                )
+                use_cp = gr.Checkbox(
+                    value=False,
+                    label='Use CP decomposition',
+                    info='A two-step approach utilizing tensor decomposition and fine-tuning to accelerate convolution layers in large neural networks, resulting in significant CPU speedups with minor accuracy drops.',
+                    visible=False,
+                )
+                decompose_both = gr.Checkbox(
+                    value=False,
+                    label='LoKr decompose both',
+                    visible=False,
+                )
+                train_on_input = gr.Checkbox(
+                    value=False,
+                    label='iA3 train on input',
+                    visible=False,
+                )
+
+            with gr.Row() as LoRA_dim_alpha:
+                network_dim = gr.Slider(
+                    minimum=1,
+                    maximum=1024,
+                    label='Network Rank (Dimension)',
+                    value=8,
+                    step=1,
+                    interactive=True,
+                )
+                network_alpha = gr.Slider(
+                    minimum=0.1,
+                    maximum=20000,
+                    label='Network Alpha',
+                    value=1,
+                    step=0.1,
+                    interactive=True,
+                    info='alpha for LoRA weight scaling',
+                )
+            with gr.Row(visible=False) as LoCon_row:
+
+                # locon= gr.Checkbox(label='Train a LoCon instead of a general LoRA (does not support v2 base models) (may not be able to some utilities now)', value=False)
+                conv_dim = gr.Slider(
+                    minimum=0,
+                    maximum=512,
+                    value=1,
+                    step=1,
+                    label='Convolution Rank (Dimension)',
+                )
+                conv_alpha = gr.Slider(
+                    minimum=0,
+                    maximum=512,
+                    value=1,
+                    step=1,
+                    label='Convolution Alpha',
+                )
+            with gr.Row():
+                scale_weight_norms = gr.Slider(
+                    label='Scale weight norms',
+                    value=0,
+                    minimum=0,
+                    maximum=1,
+                    step=0.01,
+                    info='Max Norm Regularization is a technique to stabilize network training by limiting the norm of network weights. It may be effective in suppressing overfitting of LoRA and improving stability when used with other LoRAs. See PR #545 on kohya_ss/sd_scripts repo for details.',
+                    interactive=True,
+                )
+                network_dropout = gr.Slider(
+                    label='Network dropout',
+                    value=0,
+                    minimum=0,
+                    maximum=1,
+                    step=0.01,
+                    info='Is a normal probability dropout at the neuron level. In the case of LoRA, it is applied to the output of down. Recommended range 0.1 to 0.5',
+                )
+                rank_dropout = gr.Slider(
+                    label='Rank dropout',
+                    value=0,
+                    minimum=0,
+                    maximum=1,
+                    step=0.01,
+                    info='can specify `rank_dropout` to dropout each rank with specified probability. Recommended range 0.1 to 0.3',
+                )
+                module_dropout = gr.Slider(
+                    label='Module dropout',
+                    value=0.0,
+                    minimum=0.0,
+                    maximum=1.0,
+                    step=0.01,
+                    info='can specify `module_dropout` to dropout each rank with specified probability. Recommended range 0.1 to 0.3',
+                )
+            with gr.Row(visible=False) as kohya_dylora:
+                unit = gr.Slider(
+                    minimum=1,
+                    maximum=64,
+                    label='DyLoRA Unit / Block size',
+                    value=1,
+                    step=1,
+                    interactive=True,
+                )
+
+                # Show or hide LoCon conv settings depending on LoRA type selection
+                def update_LoRA_settings(LoRA_type):
+                    log.info('LoRA type changed...')
+
+                    visibility_and_gr_types = {
+                        'LoRA_dim_alpha': (
+                            {
+                                'Kohya DyLoRA',
+                                'Kohya LoCon',
+                                'LyCORIS/DyLoRA',
+                                'LyCORIS/LoCon',
+                                'LyCORIS/LoHa',
+                                'LyCORIS/LoKr',
+                                'Standard',
+                            },
+                            gr.Row,
+                        ),
+                        'LoCon_row': (
+                            {
+                                'LoCon',
+                                'Kohya DyLoRA',
+                                'Kohya LoCon',
+                                'LyCORIS/DyLoRA',
+                                'LyCORIS/LoHa',
+                                'LyCORIS/LoKr',
+                                'LyCORIS/LoCon',
+                            },
+                            gr.Row,
+                        ),
+                        'kohya_advanced_lora': (
+                            {'Standard', 'Kohya DyLoRA', 'Kohya LoCon'},
+                            gr.Row,
+                        ),
+                        'kohya_dylora': (
+                            {'Kohya DyLoRA', 'LyCORIS/DyLoRA'},
+                            gr.Row,
+                        ),
+                        'lora_network_weights': (
+                            {'Standard', 'LoCon', 'Kohya DyLoRA', 'Kohya LoCon','LyCORIS/DyLoRA',
+                                'LyCORIS/LoHa',
+                                'LyCORIS/LoCon',
+                                'LyCORIS/LoKr',},
+                            gr.Textbox,
+                        ),
+                        'lora_network_weights_file': (
+                            {'Standard', 'LoCon', 'Kohya DyLoRA', 'Kohya LoCon','LyCORIS/DyLoRA',
+                                'LyCORIS/LoHa',
+                                'LyCORIS/LoCon',
+                                'LyCORIS/LoKr',},
+                            gr.Button,
+                        ),
+                        'dim_from_weights': (
+                            {'Standard', 'LoCon', 'Kohya DyLoRA', 'Kohya LoCon','LyCORIS/DyLoRA',
+                                'LyCORIS/LoHa',
+                                'LyCORIS/LoCon',
+                                'LyCORIS/LoKr',},
+                            gr.Checkbox,
+                        ),
+                        'factor': ({'LyCORIS/LoKr'}, gr.Slider),
+                        'use_cp': (
+                            {
+                                'LyCORIS/DyLoRA',
+                                'LyCORIS/LoHa',
+                                'LyCORIS/LoCon',
+                                'LyCORIS/LoKr',
+                            },
+                            gr.Checkbox,
+                        ),
+                        'decompose_both': ({'LyCORIS/LoKr'}, gr.Checkbox),
+                        'train_on_input': ({'LyCORIS/iA3'}, gr.Checkbox),
+                        'scale_weight_norms': (
+                            {
+                                'LoCon',
+                                'Kohya DyLoRA',
+                                'Kohya LoCon',
+                                'LyCORIS/DyLoRA',
+                                'LyCORIS/LoHa',
+                                'LyCORIS/LoCon',
+                                'LyCORIS/LoKr',
+                                'Standard',
+                            },
+                            gr.Slider,
+                        ),
+                        'network_dropout': (
+                            {
+                                'LoCon',
+                                'Kohya DyLoRA',
+                                'Kohya LoCon',
+                                'LyCORIS/DyLoRA',
+                                'LyCORIS/LoHa',
+                                'LyCORIS/LoCon',
+                                'LyCORIS/LoKr',
+                                'Standard',
+                            },
+                            gr.Slider,
+                        ),
+                        'rank_dropout': (
+                            {'LoCon', 'Kohya DyLoRA', 'Kohya LoCon',
+                                'Standard',},
+                            gr.Slider,
+                        ),
+                        'module_dropout': (
+                            {'LoCon', 'Kohya DyLoRA', 'Kohya LoCon',
+                                'Standard',},
+                            gr.Slider,
+                        ),
+                    }
+
+                    results = []
+                    for attr, (
+                        visibility,
+                        gr_type,
+                    ) in visibility_and_gr_types.items():
+                        visible = LoRA_type in visibility
+                        results.append(gr_type.update(visible=visible))
+
+                    return tuple(results)
+
+            with gr.Accordion('Advanced Configuration', open=False):
+                with gr.Row(visible=True) as kohya_advanced_lora:
+                    with gr.Tab(label='Weights'):
+                        with gr.Row(visible=True):
+                            down_lr_weight = gr.Textbox(
+                                label='Down LR weights',
+                                placeholder='(Optional) eg: 0,0,0,0,0,0,1,1,1,1,1,1',
+                                info='Specify the learning rate weight of the down blocks of U-Net.',
+                            )
+                            mid_lr_weight = gr.Textbox(
+                                label='Mid LR weights',
+                                placeholder='(Optional) eg: 0.5',
+                                info='Specify the learning rate weight of the mid block of U-Net.',
+                            )
+                            up_lr_weight = gr.Textbox(
+                                label='Up LR weights',
+                                placeholder='(Optional) eg: 0,0,0,0,0,0,1,1,1,1,1,1',
+                                info='Specify the learning rate weight of the up blocks of U-Net. The same as down_lr_weight.',
+                            )
+                            block_lr_zero_threshold = gr.Textbox(
+                                label='Blocks LR zero threshold',
+                                placeholder='(Optional) eg: 0.1',
+                                info='If the weight is not more than this value, the LoRA module is not created. The default is 0.',
+                            )
+                    with gr.Tab(label='Blocks'):
+                        with gr.Row(visible=True):
+                            block_dims = gr.Textbox(
+                                label='Block dims',
+                                placeholder='(Optional) eg: 2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2',
+                                info='Specify the dim (rank) of each block. Specify 25 numbers.',
+                            )
+                            block_alphas = gr.Textbox(
+                                label='Block alphas',
+                                placeholder='(Optional) eg: 2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2',
+                                info='Specify the alpha of each block. Specify 25 numbers as with block_dims. If omitted, the value of network_alpha is used.',
+                            )
+                    with gr.Tab(label='Conv'):
+                        with gr.Row(visible=True):
+                            conv_dims = gr.Textbox(
+                                label='Conv dims',
+                                placeholder='(Optional) eg: 2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2',
+                                info='Expand LoRA to Conv2d 3x3 and specify the dim (rank) of each block. Specify 25 numbers.',
+                            )
+                            conv_alphas = gr.Textbox(
+                                label='Conv alphas',
+                                placeholder='(Optional) eg: 2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2',
+                                info='Specify the alpha of each block when expanding LoRA to Conv2d 3x3. Specify 25 numbers. If omitted, the value of conv_alpha is used.',
+                            )
+                advanced_training = AdvancedTraining(headless=headless)
+                advanced_training.color_aug.change(
+                    color_aug_changed,
+                    inputs=[advanced_training.color_aug],
+                    outputs=[basic_training.cache_latents],
+                )
+
+            sample = SampleImages()
+
+            LoRA_type.change(
+                update_LoRA_settings,
+                inputs=[LoRA_type],
+                outputs=[
+                    LoRA_dim_alpha,
+                    LoCon_row,
+                    kohya_advanced_lora,
+                    kohya_dylora,
+                    lora_network_weights,
+                    lora_network_weights_file,
+                    dim_from_weights,
+                    factor,
+                    use_cp,
+                    decompose_both,
+                    train_on_input,
+                    scale_weight_norms,
+                    network_dropout,
+                    rank_dropout,
+                    module_dropout,
+                ],
+            )
+
+        button_run = gr.Button('Train model', variant='primary')
+
+        button_print = gr.Button('Print training command')
+
+        # Setup gradio tensorboard buttons
+        button_start_tensorboard, button_stop_tensorboard = gradio_tensorboard()
+
+        button_start_tensorboard.click(
+            start_tensorboard,
+            inputs=folders.logging_dir,
+            show_progress=False,
+        )
+
+        button_stop_tensorboard.click(
+            stop_tensorboard,
+            show_progress=False,
+        )
+
+        settings_list = [
+            source_model.pretrained_model_name_or_path,
+            source_model.v2,
+            source_model.v_parameterization,
+            source_model.sdxl_checkbox,
+            folders.logging_dir,
+            folders.train_data_dir,
+            folders.reg_data_dir,
+            folders.output_dir,
+            basic_training.max_resolution,
+            basic_training.learning_rate,
+            basic_training.lr_scheduler,
+            basic_training.lr_warmup,
+            basic_training.train_batch_size,
+            basic_training.epoch,
+            basic_training.save_every_n_epochs,
+            basic_training.mixed_precision,
+            basic_training.save_precision,
+            basic_training.seed,
+            basic_training.num_cpu_threads_per_process,
+            basic_training.cache_latents,
+            basic_training.cache_latents_to_disk,
+            basic_training.caption_extension,
+            basic_training.enable_bucket,
+            advanced_training.gradient_checkpointing,
+            advanced_training.full_fp16,
+            advanced_training.no_token_padding,
+            basic_training.stop_text_encoder_training,
+            advanced_training.xformers,
+            source_model.save_model_as,
+            advanced_training.shuffle_caption,
+            advanced_training.save_state,
+            advanced_training.resume,
+            advanced_training.prior_loss_weight,
+            text_encoder_lr,
+            unet_lr,
+            network_dim,
+            lora_network_weights,
+            dim_from_weights,
+            advanced_training.color_aug,
+            advanced_training.flip_aug,
+            advanced_training.clip_skip,
+            advanced_training.gradient_accumulation_steps,
+            advanced_training.mem_eff_attn,
+            folders.output_name,
+            source_model.model_list,
+            advanced_training.max_token_length,
+            advanced_training.max_train_epochs,
+            advanced_training.max_data_loader_n_workers,
+            network_alpha,
+            folders.training_comment,
+            advanced_training.keep_tokens,
+            advanced_training.lr_scheduler_num_cycles,
+            advanced_training.lr_scheduler_power,
+            advanced_training.persistent_data_loader_workers,
+            advanced_training.bucket_no_upscale,
+            advanced_training.random_crop,
+            advanced_training.bucket_reso_steps,
+            advanced_training.caption_dropout_every_n_epochs,
+            advanced_training.caption_dropout_rate,
+            basic_training.optimizer,
+            basic_training.optimizer_args,
+            advanced_training.noise_offset_type,
+            advanced_training.noise_offset,
+            advanced_training.adaptive_noise_scale,
+            advanced_training.multires_noise_iterations,
+            advanced_training.multires_noise_discount,
+            LoRA_type,
+            factor,
+            use_cp,
+            decompose_both,
+            train_on_input,
+            conv_dim,
+            conv_alpha,
+            sample.sample_every_n_steps,
+            sample.sample_every_n_epochs,
+            sample.sample_sampler,
+            sample.sample_prompts,
+            advanced_training.additional_parameters,
+            advanced_training.vae_batch_size,
+            advanced_training.min_snr_gamma,
+            down_lr_weight,
+            mid_lr_weight,
+            up_lr_weight,
+            block_lr_zero_threshold,
+            block_dims,
+            block_alphas,
+            conv_dims,
+            conv_alphas,
+            advanced_training.weighted_captions,
+            unit,
+            advanced_training.save_every_n_steps,
+            advanced_training.save_last_n_steps,
+            advanced_training.save_last_n_steps_state,
+            advanced_training.use_wandb,
+            advanced_training.wandb_api_key,
+            advanced_training.scale_v_pred_loss_like_noise_pred,
+            scale_weight_norms,
+            network_dropout,
+            rank_dropout,
+            module_dropout,
+            sdxl_params.sdxl_cache_text_encoder_outputs,
+            sdxl_params.sdxl_no_half_vae,
+            advanced_training.min_timestep,
+            advanced_training.max_timestep,
+        ]
+
+        config.button_open_config.click(
+            open_configuration,
+            inputs=[dummy_db_true, dummy_db_false, config.config_file_name]
+            + settings_list
+            + [training_preset],
+            outputs=[config.config_file_name]
+            + settings_list
+            + [training_preset, LoCon_row],
+            show_progress=False,
+        )
+
+        config.button_load_config.click(
+            open_configuration,
+            inputs=[dummy_db_false, dummy_db_false, config.config_file_name]
+            + settings_list
+            + [training_preset],
+            outputs=[config.config_file_name]
+            + settings_list
+            + [training_preset, LoCon_row],
+            show_progress=False,
+        )
+
+        training_preset.input(
+            open_configuration,
+            inputs=[dummy_db_false, dummy_db_true, config.config_file_name]
+            + settings_list
+            + [training_preset],
+            outputs=[gr.Textbox()] + settings_list + [training_preset, LoCon_row],
+            show_progress=False,
+        )
+
+        config.button_save_config.click(
+            save_configuration,
+            inputs=[dummy_db_false, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name],
+            show_progress=False,
+        )
+
+        config.button_save_as_config.click(
+            save_configuration,
+            inputs=[dummy_db_true, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name],
+            show_progress=False,
+        )
+
+        button_run.click(
+            train_model,
+            inputs=[dummy_headless] + [dummy_db_false] + settings_list,
+            show_progress=False,
+        )
+
+        button_print.click(
+            train_model,
+            inputs=[dummy_headless] + [dummy_db_true] + settings_list,
+            show_progress=False,
+        )
+        
+    with gr.Tab('Tools'):
+        lora_tools = LoRATools(folders=folders, headless=headless)
+        
+    with gr.Tab('Guides'):
+        gr.Markdown(
+            'This section provide Various LoRA guides and information...'
+        )
+        if os.path.exists('./docs/LoRA/top_level.md'):
+            with open(os.path.join('./docs/LoRA/top_level.md'), 'r', encoding='utf8') as file:
+                guides_top_level = file.read() + '\n'
+        gr.Markdown(guides_top_level)
+
+    return (
+        folders.train_data_dir,
+        folders.reg_data_dir,
+        folders.output_dir,
+        folders.logging_dir,
+    )
+
+
+def UI(**kwargs):
+    css = ''
+
+    headless = kwargs.get('headless', False)
+    log.info(f'headless: {headless}')
+
+    if os.path.exists('./style.css'):
+        with open(os.path.join('./style.css'), 'r', encoding='utf8') as file:
+            log.info('Load CSS...')
+            css += file.read() + '\n'
+
+    interface = gr.Blocks(
+        css=css, title='Kohya_ss GUI', theme=gr.themes.Default()
+    )
+
+    with interface:
+        with gr.Tab('LoRA'):
+            (
+                train_data_dir_input,
+                reg_data_dir_input,
+                output_dir_input,
+                logging_dir_input,
+            ) = lora_tab(headless=headless)
+        with gr.Tab('Utilities'):
+            utilities_tab(
+                train_data_dir_input=train_data_dir_input,
+                reg_data_dir_input=reg_data_dir_input,
+                output_dir_input=output_dir_input,
+                logging_dir_input=logging_dir_input,
+                enable_copy_info_button=True,
+                headless=headless,
+            )
+
+    # Show the interface
+    launch_kwargs = {}
+    username = kwargs.get('username')
+    password = kwargs.get('password')
+    server_port = kwargs.get('server_port', 0)
+    inbrowser = kwargs.get('inbrowser', False)
+    share = kwargs.get('share', False)
+    server_name = kwargs.get('listen')
+
+    launch_kwargs['server_name'] = server_name
+    if username and password:
+        launch_kwargs['auth'] = (username, password)
+    if server_port > 0:
+        launch_kwargs['server_port'] = server_port
+    if inbrowser:
+        launch_kwargs['inbrowser'] = inbrowser
+    if share:
+        launch_kwargs['share'] = share
+    log.info(launch_kwargs)
+    interface.launch(**launch_kwargs)
+
+
+if __name__ == '__main__':
+    # torch.cuda.set_per_process_memory_fraction(0.48)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--listen',
+        type=str,
+        default='127.0.0.1',
+        help='IP to listen on for connections to Gradio',
+    )
+    parser.add_argument(
+        '--username', type=str, default='', help='Username for authentication'
+    )
+    parser.add_argument(
+        '--password', type=str, default='', help='Password for authentication'
+    )
+    parser.add_argument(
+        '--server_port',
+        type=int,
+        default=0,
+        help='Port to run the server listener on',
+    )
+    parser.add_argument(
+        '--inbrowser', action='store_true', help='Open in browser'
+    )
+    parser.add_argument(
+        '--share', action='store_true', help='Share the gradio UI'
+    )
+    parser.add_argument(
+        '--headless', action='store_true', help='Is the server headless'
+    )
+
+    args = parser.parse_args()
+
+    UI(
+        username=args.username,
+        password=args.password,
+        inbrowser=args.inbrowser,
+        server_port=args.server_port,
+        share=args.share,
+        listen=args.listen,
+        headless=args.headless,
+    )
diff --git a/networks/check_lora_weights.py b/networks/check_lora_weights.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb8dcd6ba4f393fe0a40ae668c530e18b87aea16
--- /dev/null
+++ b/networks/check_lora_weights.py
@@ -0,0 +1,39 @@
+import argparse
+import os
+import torch
+from safetensors.torch import load_file
+
+
+def main(file):
+  print(f"loading: {file}")
+  if os.path.splitext(file)[1] == '.safetensors':
+    sd = load_file(file)
+  else:
+    sd = torch.load(file, map_location='cpu')
+
+  values = []
+
+  keys = list(sd.keys())
+  for key in keys:
+    if 'lora_up' in key or 'lora_down' in key:
+      values.append((key, sd[key]))
+  print(f"number of LoRA modules: {len(values)}")
+
+  for key, value in values:
+    value = value.to(torch.float32)
+    print(f"{key},{str(tuple(value.size())).replace(', ', '-')},{torch.mean(torch.abs(value))},{torch.min(torch.abs(value))}")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("file", type=str, help="model file to check / 重みを確認するモデルファイル")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+
+  main(args.file)
diff --git a/networks/dylora.py b/networks/dylora.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5a55d1988ac4bc35f1d1657792efcfbaaccea9e
--- /dev/null
+++ b/networks/dylora.py
@@ -0,0 +1,450 @@
+# some codes are copied from:
+# https://github.com/huawei-noah/KD-NLP/blob/main/DyLoRA/
+
+# Copyright (C) 2022. Huawei Technologies Co., Ltd. All rights reserved.
+# Changes made to the original code:
+# 2022.08.20 - Integrate the DyLoRA layer for the LoRA Linear layer
+#  ------------------------------------------------------------------------------------------
+#  Copyright (c) Microsoft Corporation. All rights reserved.
+#  Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+#  ------------------------------------------------------------------------------------------
+
+import math
+import os
+import random
+from typing import List, Tuple, Union
+import torch
+from torch import nn
+
+
+class DyLoRAModule(torch.nn.Module):
+    """
+    replaces forward method of the original Linear, instead of replacing the original Linear module.
+    """
+
+    # NOTE: support dropout in future
+    def __init__(self, lora_name, org_module: torch.nn.Module, multiplier=1.0, lora_dim=4, alpha=1, unit=1):
+        super().__init__()
+        self.lora_name = lora_name
+        self.lora_dim = lora_dim
+        self.unit = unit
+        assert self.lora_dim % self.unit == 0, "rank must be a multiple of unit"
+
+        if org_module.__class__.__name__ == "Conv2d":
+            in_dim = org_module.in_channels
+            out_dim = org_module.out_channels
+        else:
+            in_dim = org_module.in_features
+            out_dim = org_module.out_features
+
+        if type(alpha) == torch.Tensor:
+            alpha = alpha.detach().float().numpy()  # without casting, bf16 causes error
+        alpha = self.lora_dim if alpha is None or alpha == 0 else alpha
+        self.scale = alpha / self.lora_dim
+        self.register_buffer("alpha", torch.tensor(alpha))  # 定数として扱える
+
+        self.is_conv2d = org_module.__class__.__name__ == "Conv2d"
+        self.is_conv2d_3x3 = self.is_conv2d and org_module.kernel_size == (3, 3)
+
+        if self.is_conv2d and self.is_conv2d_3x3:
+            kernel_size = org_module.kernel_size
+            self.stride = org_module.stride
+            self.padding = org_module.padding
+            self.lora_A = nn.ParameterList([org_module.weight.new_zeros((1, in_dim, *kernel_size)) for _ in range(self.lora_dim)])
+            self.lora_B = nn.ParameterList([org_module.weight.new_zeros((out_dim, 1, 1, 1)) for _ in range(self.lora_dim)])
+        else:
+            self.lora_A = nn.ParameterList([org_module.weight.new_zeros((1, in_dim)) for _ in range(self.lora_dim)])
+            self.lora_B = nn.ParameterList([org_module.weight.new_zeros((out_dim, 1)) for _ in range(self.lora_dim)])
+
+        # same as microsoft's
+        for lora in self.lora_A:
+            torch.nn.init.kaiming_uniform_(lora, a=math.sqrt(5))
+        for lora in self.lora_B:
+            torch.nn.init.zeros_(lora)
+
+        self.multiplier = multiplier
+        self.org_module = org_module  # remove in applying
+
+    def apply_to(self):
+        self.org_forward = self.org_module.forward
+        self.org_module.forward = self.forward
+        del self.org_module
+
+    def forward(self, x):
+        result = self.org_forward(x)
+
+        # specify the dynamic rank
+        trainable_rank = random.randint(0, self.lora_dim - 1)
+        trainable_rank = trainable_rank - trainable_rank % self.unit  # make sure the rank is a multiple of unit
+
+        # 一部のパラメータを固定して、残りのパラメータを学習する
+        for i in range(0, trainable_rank):
+            self.lora_A[i].requires_grad = False
+            self.lora_B[i].requires_grad = False
+        for i in range(trainable_rank, trainable_rank + self.unit):
+            self.lora_A[i].requires_grad = True
+            self.lora_B[i].requires_grad = True
+        for i in range(trainable_rank + self.unit, self.lora_dim):
+            self.lora_A[i].requires_grad = False
+            self.lora_B[i].requires_grad = False
+
+        lora_A = torch.cat(tuple(self.lora_A), dim=0)
+        lora_B = torch.cat(tuple(self.lora_B), dim=1)
+
+        # calculate with lora_A and lora_B
+        if self.is_conv2d_3x3:
+            ab = torch.nn.functional.conv2d(x, lora_A, stride=self.stride, padding=self.padding)
+            ab = torch.nn.functional.conv2d(ab, lora_B)
+        else:
+            ab = x
+            if self.is_conv2d:
+                ab = ab.reshape(ab.size(0), ab.size(1), -1).transpose(1, 2)  # (N, C, H, W) -> (N, H*W, C)
+
+            ab = torch.nn.functional.linear(ab, lora_A)
+            ab = torch.nn.functional.linear(ab, lora_B)
+
+            if self.is_conv2d:
+                ab = ab.transpose(1, 2).reshape(ab.size(0), -1, *x.size()[2:])  # (N, H*W, C) -> (N, C, H, W)
+
+        # 最後の項は、低rankをより大きくするためのスケーリング（じゃないかな）
+        result = result + ab * self.scale * math.sqrt(self.lora_dim / (trainable_rank + self.unit))
+
+        # NOTE weightに加算してからlinear/conv2dを呼んだほうが速いかも
+        return result
+
+    def state_dict(self, destination=None, prefix="", keep_vars=False):
+        # state dictを通常のLoRAと同じにする:
+        # nn.ParameterListは `.lora_A.0` みたいな名前になるので、forwardと同様にcatして入れ替える
+        sd = super().state_dict(destination=destination, prefix=prefix, keep_vars=keep_vars)
+
+        lora_A_weight = torch.cat(tuple(self.lora_A), dim=0)
+        if self.is_conv2d and not self.is_conv2d_3x3:
+            lora_A_weight = lora_A_weight.unsqueeze(-1).unsqueeze(-1)
+
+        lora_B_weight = torch.cat(tuple(self.lora_B), dim=1)
+        if self.is_conv2d and not self.is_conv2d_3x3:
+            lora_B_weight = lora_B_weight.unsqueeze(-1).unsqueeze(-1)
+
+        sd[self.lora_name + ".lora_down.weight"] = lora_A_weight if keep_vars else lora_A_weight.detach()
+        sd[self.lora_name + ".lora_up.weight"] = lora_B_weight if keep_vars else lora_B_weight.detach()
+
+        i = 0
+        while True:
+            key_a = f"{self.lora_name}.lora_A.{i}"
+            key_b = f"{self.lora_name}.lora_B.{i}"
+            if key_a in sd:
+                sd.pop(key_a)
+                sd.pop(key_b)
+            else:
+                break
+            i += 1
+        return sd
+
+    def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
+        # 通常のLoRAと同じstate dictを読み込めるようにする：この方法はchatGPTに聞いた
+        lora_A_weight = state_dict.pop(self.lora_name + ".lora_down.weight", None)
+        lora_B_weight = state_dict.pop(self.lora_name + ".lora_up.weight", None)
+
+        if lora_A_weight is None or lora_B_weight is None:
+            if strict:
+                raise KeyError(f"{self.lora_name}.lora_down/up.weight is not found")
+            else:
+                return
+
+        if self.is_conv2d and not self.is_conv2d_3x3:
+            lora_A_weight = lora_A_weight.squeeze(-1).squeeze(-1)
+            lora_B_weight = lora_B_weight.squeeze(-1).squeeze(-1)
+
+        state_dict.update(
+            {f"{self.lora_name}.lora_A.{i}": nn.Parameter(lora_A_weight[i].unsqueeze(0)) for i in range(lora_A_weight.size(0))}
+        )
+        state_dict.update(
+            {f"{self.lora_name}.lora_B.{i}": nn.Parameter(lora_B_weight[:, i].unsqueeze(1)) for i in range(lora_B_weight.size(1))}
+        )
+
+        super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)
+
+
+def create_network(multiplier, network_dim, network_alpha, vae, text_encoder, unet, **kwargs):
+    if network_dim is None:
+        network_dim = 4  # default
+    if network_alpha is None:
+        network_alpha = 1.0
+
+    # extract dim/alpha for conv2d, and block dim
+    conv_dim = kwargs.get("conv_dim", None)
+    conv_alpha = kwargs.get("conv_alpha", None)
+    unit = kwargs.get("unit", None)
+    if conv_dim is not None:
+        conv_dim = int(conv_dim)
+        assert conv_dim == network_dim, "conv_dim must be same as network_dim"
+        if conv_alpha is None:
+            conv_alpha = 1.0
+        else:
+            conv_alpha = float(conv_alpha)
+    if unit is not None:
+        unit = int(unit)
+    else:
+        unit = 1
+
+    network = DyLoRANetwork(
+        text_encoder,
+        unet,
+        multiplier=multiplier,
+        lora_dim=network_dim,
+        alpha=network_alpha,
+        apply_to_conv=conv_dim is not None,
+        unit=unit,
+        varbose=True,
+    )
+    return network
+
+
+# Create network from weights for inference, weights are not loaded here (because can be merged)
+def create_network_from_weights(multiplier, file, vae, text_encoder, unet, weights_sd=None, for_inference=False, **kwargs):
+    if weights_sd is None:
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import load_file, safe_open
+
+            weights_sd = load_file(file)
+        else:
+            weights_sd = torch.load(file, map_location="cpu")
+
+    # get dim/alpha mapping
+    modules_dim = {}
+    modules_alpha = {}
+    for key, value in weights_sd.items():
+        if "." not in key:
+            continue
+
+        lora_name = key.split(".")[0]
+        if "alpha" in key:
+            modules_alpha[lora_name] = value
+        elif "lora_down" in key:
+            dim = value.size()[0]
+            modules_dim[lora_name] = dim
+            # print(lora_name, value.size(), dim)
+
+    # support old LoRA without alpha
+    for key in modules_dim.keys():
+        if key not in modules_alpha:
+            modules_alpha = modules_dim[key]
+
+    module_class = DyLoRAModule
+
+    network = DyLoRANetwork(
+        text_encoder, unet, multiplier=multiplier, modules_dim=modules_dim, modules_alpha=modules_alpha, module_class=module_class
+    )
+    return network, weights_sd
+
+
+class DyLoRANetwork(torch.nn.Module):
+    UNET_TARGET_REPLACE_MODULE = ["Transformer2DModel"]
+    UNET_TARGET_REPLACE_MODULE_CONV2D_3X3 = ["ResnetBlock2D", "Downsample2D", "Upsample2D"]
+    TEXT_ENCODER_TARGET_REPLACE_MODULE = ["CLIPAttention", "CLIPMLP"]
+    LORA_PREFIX_UNET = "lora_unet"
+    LORA_PREFIX_TEXT_ENCODER = "lora_te"
+
+    def __init__(
+        self,
+        text_encoder,
+        unet,
+        multiplier=1.0,
+        lora_dim=4,
+        alpha=1,
+        apply_to_conv=False,
+        modules_dim=None,
+        modules_alpha=None,
+        unit=1,
+        module_class=DyLoRAModule,
+        varbose=False,
+    ) -> None:
+        super().__init__()
+        self.multiplier = multiplier
+
+        self.lora_dim = lora_dim
+        self.alpha = alpha
+        self.apply_to_conv = apply_to_conv
+
+        if modules_dim is not None:
+            print(f"create LoRA network from weights")
+        else:
+            print(f"create LoRA network. base dim (rank): {lora_dim}, alpha: {alpha}, unit: {unit}")
+            if self.apply_to_conv:
+                print(f"apply LoRA to Conv2d with kernel size (3,3).")
+
+        # create module instances
+        def create_modules(is_unet, root_module: torch.nn.Module, target_replace_modules) -> List[DyLoRAModule]:
+            prefix = DyLoRANetwork.LORA_PREFIX_UNET if is_unet else DyLoRANetwork.LORA_PREFIX_TEXT_ENCODER
+            loras = []
+            for name, module in root_module.named_modules():
+                if module.__class__.__name__ in target_replace_modules:
+                    for child_name, child_module in module.named_modules():
+                        is_linear = child_module.__class__.__name__ == "Linear"
+                        is_conv2d = child_module.__class__.__name__ == "Conv2d"
+                        is_conv2d_1x1 = is_conv2d and child_module.kernel_size == (1, 1)
+
+                        if is_linear or is_conv2d:
+                            lora_name = prefix + "." + name + "." + child_name
+                            lora_name = lora_name.replace(".", "_")
+
+                            dim = None
+                            alpha = None
+                            if modules_dim is not None:
+                                if lora_name in modules_dim:
+                                    dim = modules_dim[lora_name]
+                                    alpha = modules_alpha[lora_name]
+                            else:
+                                if is_linear or is_conv2d_1x1 or apply_to_conv:
+                                    dim = self.lora_dim
+                                    alpha = self.alpha
+
+                            if dim is None or dim == 0:
+                                continue
+
+                            # dropout and fan_in_fan_out is default
+                            lora = module_class(lora_name, child_module, self.multiplier, dim, alpha, unit)
+                            loras.append(lora)
+            return loras
+
+        self.text_encoder_loras = create_modules(False, text_encoder, DyLoRANetwork.TEXT_ENCODER_TARGET_REPLACE_MODULE)
+        print(f"create LoRA for Text Encoder: {len(self.text_encoder_loras)} modules.")
+
+        # extend U-Net target modules if conv2d 3x3 is enabled, or load from weights
+        target_modules = DyLoRANetwork.UNET_TARGET_REPLACE_MODULE
+        if modules_dim is not None or self.apply_to_conv:
+            target_modules += DyLoRANetwork.UNET_TARGET_REPLACE_MODULE_CONV2D_3X3
+
+        self.unet_loras = create_modules(True, unet, target_modules)
+        print(f"create LoRA for U-Net: {len(self.unet_loras)} modules.")
+
+    def set_multiplier(self, multiplier):
+        self.multiplier = multiplier
+        for lora in self.text_encoder_loras + self.unet_loras:
+            lora.multiplier = self.multiplier
+
+    def load_weights(self, file):
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import load_file
+
+            weights_sd = load_file(file)
+        else:
+            weights_sd = torch.load(file, map_location="cpu")
+
+        info = self.load_state_dict(weights_sd, False)
+        return info
+
+    def apply_to(self, text_encoder, unet, apply_text_encoder=True, apply_unet=True):
+        if apply_text_encoder:
+            print("enable LoRA for text encoder")
+        else:
+            self.text_encoder_loras = []
+
+        if apply_unet:
+            print("enable LoRA for U-Net")
+        else:
+            self.unet_loras = []
+
+        for lora in self.text_encoder_loras + self.unet_loras:
+            lora.apply_to()
+            self.add_module(lora.lora_name, lora)
+
+    """
+    def merge_to(self, text_encoder, unet, weights_sd, dtype, device):
+        apply_text_encoder = apply_unet = False
+        for key in weights_sd.keys():
+            if key.startswith(DyLoRANetwork.LORA_PREFIX_TEXT_ENCODER):
+                apply_text_encoder = True
+            elif key.startswith(DyLoRANetwork.LORA_PREFIX_UNET):
+                apply_unet = True
+
+        if apply_text_encoder:
+            print("enable LoRA for text encoder")
+        else:
+            self.text_encoder_loras = []
+
+        if apply_unet:
+            print("enable LoRA for U-Net")
+        else:
+            self.unet_loras = []
+
+        for lora in self.text_encoder_loras + self.unet_loras:
+            sd_for_lora = {}
+            for key in weights_sd.keys():
+                if key.startswith(lora.lora_name):
+                    sd_for_lora[key[len(lora.lora_name) + 1 :]] = weights_sd[key]
+            lora.merge_to(sd_for_lora, dtype, device)
+
+        print(f"weights are merged")
+    """
+
+    def prepare_optimizer_params(self, text_encoder_lr, unet_lr, default_lr):
+        self.requires_grad_(True)
+        all_params = []
+
+        def enumerate_params(loras):
+            params = []
+            for lora in loras:
+                params.extend(lora.parameters())
+            return params
+
+        if self.text_encoder_loras:
+            param_data = {"params": enumerate_params(self.text_encoder_loras)}
+            if text_encoder_lr is not None:
+                param_data["lr"] = text_encoder_lr
+            all_params.append(param_data)
+
+        if self.unet_loras:
+            param_data = {"params": enumerate_params(self.unet_loras)}
+            if unet_lr is not None:
+                param_data["lr"] = unet_lr
+            all_params.append(param_data)
+
+        return all_params
+
+    def enable_gradient_checkpointing(self):
+        # not supported
+        pass
+
+    def prepare_grad_etc(self, text_encoder, unet):
+        self.requires_grad_(True)
+
+    def on_epoch_start(self, text_encoder, unet):
+        self.train()
+
+    def get_trainable_params(self):
+        return self.parameters()
+
+    def save_weights(self, file, dtype, metadata):
+        if metadata is not None and len(metadata) == 0:
+            metadata = None
+
+        state_dict = self.state_dict()
+
+        if dtype is not None:
+            for key in list(state_dict.keys()):
+                v = state_dict[key]
+                v = v.detach().clone().to("cpu").to(dtype)
+                state_dict[key] = v
+
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import save_file
+            from library import train_util
+
+            # Precalculate model hashes to save time on indexing
+            if metadata is None:
+                metadata = {}
+            model_hash, legacy_hash = train_util.precalculate_safetensors_hashes(state_dict, metadata)
+            metadata["sshs_model_hash"] = model_hash
+            metadata["sshs_legacy_hash"] = legacy_hash
+
+            save_file(state_dict, file, metadata)
+        else:
+            torch.save(state_dict, file)
+
+    # mask is a tensor with values from 0 to 1
+    def set_region(self, sub_prompt_index, is_last_network, mask):
+        pass
+
+    def set_current_generation(self, batch_size, num_sub_prompts, width, height, shared):
+        pass
diff --git a/networks/extract_lora_from_dylora.py b/networks/extract_lora_from_dylora.py
new file mode 100644
index 0000000000000000000000000000000000000000..0abee98368911575c6e84974ab7ca7a4cdc97e0e
--- /dev/null
+++ b/networks/extract_lora_from_dylora.py
@@ -0,0 +1,125 @@
+# Convert LoRA to different rank approximation (should only be used to go to lower rank)
+# This code is based off the extract_lora_from_models.py file which is based on https://github.com/cloneofsimo/lora/blob/develop/lora_diffusion/cli_svd.py
+# Thanks to cloneofsimo
+
+import argparse
+import math
+import os
+import torch
+from safetensors.torch import load_file, save_file, safe_open
+from tqdm import tqdm
+from library import train_util, model_util
+import numpy as np
+
+
+def load_state_dict(file_name):
+    if model_util.is_safetensors(file_name):
+        sd = load_file(file_name)
+        with safe_open(file_name, framework="pt") as f:
+            metadata = f.metadata()
+    else:
+        sd = torch.load(file_name, map_location="cpu")
+        metadata = None
+
+    return sd, metadata
+
+
+def save_to_file(file_name, model, metadata):
+    if model_util.is_safetensors(file_name):
+        save_file(model, file_name, metadata)
+    else:
+        torch.save(model, file_name)
+
+
+def split_lora_model(lora_sd, unit):
+    max_rank = 0
+
+    # Extract loaded lora dim and alpha
+    for key, value in lora_sd.items():
+        if "lora_down" in key:
+            rank = value.size()[0]
+            if rank > max_rank:
+                max_rank = rank
+    print(f"Max rank: {max_rank}")
+
+    rank = unit
+    split_models = []
+    new_alpha = None
+    while rank < max_rank:
+        print(f"Splitting rank {rank}")
+        new_sd = {}
+        for key, value in lora_sd.items():
+            if "lora_down" in key:
+                new_sd[key] = value[:rank].contiguous()
+            elif "lora_up" in key:
+                new_sd[key] = value[:, :rank].contiguous()
+            else:
+                # なぜかscaleするとおかしくなる……
+                # this_rank = lora_sd[key.replace("alpha", "lora_down.weight")].size()[0]
+                # scale = math.sqrt(this_rank / rank)  # rank is > unit
+                # print(key, value.size(), this_rank, rank, value, scale)
+                # new_alpha = value * scale  # always same
+                # new_sd[key] = new_alpha
+                new_sd[key] = value
+
+        split_models.append((new_sd, rank, new_alpha))
+        rank += unit
+
+    return max_rank, split_models
+
+
+def split(args):
+    print("loading Model...")
+    lora_sd, metadata = load_state_dict(args.model)
+
+    print("Splitting Model...")
+    original_rank, split_models = split_lora_model(lora_sd, args.unit)
+
+    comment = metadata.get("ss_training_comment", "")
+    for state_dict, new_rank, new_alpha in split_models:
+        # update metadata
+        if metadata is None:
+            new_metadata = {}
+        else:
+            new_metadata = metadata.copy()
+
+        new_metadata["ss_training_comment"] = f"split from DyLoRA, rank {original_rank} to {new_rank}; {comment}"
+        new_metadata["ss_network_dim"] = str(new_rank)
+        # new_metadata["ss_network_alpha"] = str(new_alpha.float().numpy())
+
+        model_hash, legacy_hash = train_util.precalculate_safetensors_hashes(state_dict, metadata)
+        metadata["sshs_model_hash"] = model_hash
+        metadata["sshs_legacy_hash"] = legacy_hash
+
+        filename, ext = os.path.splitext(args.save_to)
+        model_file_name = filename + f"-{new_rank:04d}{ext}"
+
+        print(f"saving model to: {model_file_name}")
+        save_to_file(model_file_name, state_dict, new_metadata)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--unit", type=int, default=None, help="size of rank to split into / rankを分割するサイズ")
+    parser.add_argument(
+        "--save_to",
+        type=str,
+        default=None,
+        help="destination base file name: ckpt or safetensors file / 保存先のファイル名のbase、ckptまたはsafetensors",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="DyLoRA model to resize at to new rank: ckpt or safetensors file / 読み込むDyLoRAモデル、ckptまたはsafetensors",
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    split(args)
diff --git a/networks/extract_lora_from_models.py b/networks/extract_lora_from_models.py
new file mode 100644
index 0000000000000000000000000000000000000000..f001e7eb29d68c0fdfa58a59743b27b911b20d2b
--- /dev/null
+++ b/networks/extract_lora_from_models.py
@@ -0,0 +1,189 @@
+# extract approximating LoRA by svd from two SD models
+# The code is based on https://github.com/cloneofsimo/lora/blob/develop/lora_diffusion/cli_svd.py
+# Thanks to cloneofsimo!
+
+import argparse
+import os
+import torch
+from safetensors.torch import load_file, save_file
+from tqdm import tqdm
+import library.model_util as model_util
+import lora
+
+
+CLAMP_QUANTILE = 0.99
+MIN_DIFF = 1e-6
+
+
+def save_to_file(file_name, model, state_dict, dtype):
+  if dtype is not None:
+    for key in list(state_dict.keys()):
+      if type(state_dict[key]) == torch.Tensor:
+        state_dict[key] = state_dict[key].to(dtype)
+
+  if os.path.splitext(file_name)[1] == '.safetensors':
+    save_file(model, file_name)
+  else:
+    torch.save(model, file_name)
+
+
+def svd(args):
+  def str_to_dtype(p):
+    if p == 'float':
+      return torch.float
+    if p == 'fp16':
+      return torch.float16
+    if p == 'bf16':
+      return torch.bfloat16
+    return None
+
+  save_dtype = str_to_dtype(args.save_precision)
+
+  print(f"loading SD model : {args.model_org}")
+  text_encoder_o, _, unet_o = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_org)
+  print(f"loading SD model : {args.model_tuned}")
+  text_encoder_t, _, unet_t = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_tuned)
+
+  # create LoRA network to extract weights: Use dim (rank) as alpha
+  if args.conv_dim is None:
+    kwargs = {}
+  else:
+    kwargs = {"conv_dim": args.conv_dim, "conv_alpha": args.conv_dim}
+
+  lora_network_o = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_o, unet_o, **kwargs)
+  lora_network_t = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_t, unet_t, **kwargs)
+  assert len(lora_network_o.text_encoder_loras) == len(
+      lora_network_t.text_encoder_loras), f"model version is different (SD1.x vs SD2.x) / それぞれのモデルのバージョンが違います（SD1.xベースとSD2.xベース） "
+
+  # get diffs
+  diffs = {}
+  text_encoder_different = False
+  for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.text_encoder_loras, lora_network_t.text_encoder_loras)):
+    lora_name = lora_o.lora_name
+    module_o = lora_o.org_module
+    module_t = lora_t.org_module
+    diff = module_t.weight - module_o.weight
+
+    # Text Encoder might be same
+    if torch.max(torch.abs(diff)) > MIN_DIFF:
+      text_encoder_different = True
+
+    diff = diff.float()
+    diffs[lora_name] = diff
+
+  if not text_encoder_different:
+    print("Text encoder is same. Extract U-Net only.")
+    lora_network_o.text_encoder_loras = []
+    diffs = {}
+
+  for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.unet_loras, lora_network_t.unet_loras)):
+    lora_name = lora_o.lora_name
+    module_o = lora_o.org_module
+    module_t = lora_t.org_module
+    diff = module_t.weight - module_o.weight
+    diff = diff.float()
+
+    if args.device:
+      diff = diff.to(args.device)
+
+    diffs[lora_name] = diff
+
+  # make LoRA with svd
+  print("calculating by svd")
+  lora_weights = {}
+  with torch.no_grad():
+    for lora_name, mat in tqdm(list(diffs.items())):
+      # if args.conv_dim is None, diffs do not include LoRAs for conv2d-3x3
+      conv2d = (len(mat.size()) == 4)
+      kernel_size = None if not conv2d else mat.size()[2:4]
+      conv2d_3x3 = conv2d and kernel_size != (1, 1)
+
+      rank = args.dim if not conv2d_3x3 or args.conv_dim is None else args.conv_dim
+      out_dim, in_dim = mat.size()[0:2]
+
+      if args.device:
+        mat = mat.to(args.device)
+
+      # print(lora_name, mat.size(), mat.device, rank, in_dim, out_dim)
+      rank = min(rank, in_dim, out_dim)                           # LoRA rank cannot exceed the original dim
+
+      if conv2d:
+        if conv2d_3x3:
+          mat = mat.flatten(start_dim=1)
+        else:
+          mat = mat.squeeze()
+
+      U, S, Vh = torch.linalg.svd(mat)
+
+      U = U[:, :rank]
+      S = S[:rank]
+      U = U @ torch.diag(S)
+
+      Vh = Vh[:rank, :]
+
+      dist = torch.cat([U.flatten(), Vh.flatten()])
+      hi_val = torch.quantile(dist, CLAMP_QUANTILE)
+      low_val = -hi_val
+
+      U = U.clamp(low_val, hi_val)
+      Vh = Vh.clamp(low_val, hi_val)
+
+      if conv2d:
+        U = U.reshape(out_dim, rank, 1, 1)
+        Vh = Vh.reshape(rank, in_dim, kernel_size[0], kernel_size[1])
+
+      U = U.to("cpu").contiguous()
+      Vh = Vh.to("cpu").contiguous()
+
+      lora_weights[lora_name] = (U, Vh)
+
+  # make state dict for LoRA
+  lora_sd = {}
+  for lora_name, (up_weight, down_weight) in lora_weights.items():
+    lora_sd[lora_name + '.lora_up.weight'] = up_weight
+    lora_sd[lora_name + '.lora_down.weight'] = down_weight
+    lora_sd[lora_name + '.alpha'] = torch.tensor(down_weight.size()[0])
+
+  # load state dict to LoRA and save it
+  lora_network_save, lora_sd = lora.create_network_from_weights(1.0, None, None, text_encoder_o, unet_o, weights_sd=lora_sd)
+  lora_network_save.apply_to(text_encoder_o, unet_o)  # create internal module references for state_dict  
+
+  info = lora_network_save.load_state_dict(lora_sd)
+  print(f"Loading extracted LoRA weights: {info}")
+
+  dir_name = os.path.dirname(args.save_to)
+  if dir_name and not os.path.exists(dir_name):
+    os.makedirs(dir_name, exist_ok=True)
+
+  # minimum metadata
+  metadata = {"ss_network_module": "networks.lora", "ss_network_dim": str(args.dim), "ss_network_alpha": str(args.dim)}
+
+  lora_network_save.save_weights(args.save_to, save_dtype, metadata)
+  print(f"LoRA weights are saved to: {args.save_to}")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--v2", action='store_true',
+                      help='load Stable Diffusion v2.x model / Stable Diffusion 2.xのモデルを読み込む')
+  parser.add_argument("--save_precision", type=str, default=None,
+                      choices=[None, "float", "fp16", "bf16"], help="precision in saving, same to merging if omitted / 保存時に精度を変更して保存する、省略時はfloat")
+  parser.add_argument("--model_org", type=str, default=None,
+                      help="Stable Diffusion original model: ckpt or safetensors file / 元モデル、ckptまたはsafetensors")
+  parser.add_argument("--model_tuned", type=str, default=None,
+                      help="Stable Diffusion tuned model, LoRA is difference of `original to tuned`: ckpt or safetensors file / 派生モデル（生成されるLoRAは元→派生の差分になります）、ckptまたはsafetensors")
+  parser.add_argument("--save_to", type=str, default=None,
+                      help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors")
+  parser.add_argument("--dim", type=int, default=4, help="dimension (rank) of LoRA (default 4) / LoRAの次元数（rank）（デフォルト4）")
+  parser.add_argument("--conv_dim", type=int, default=None,
+                      help="dimension (rank) of LoRA for Conv2d-3x3 (default None, disabled) / LoRAのConv2d-3x3の次元数（rank）（デフォルトNone、適用なし）")
+  parser.add_argument("--device", type=str, default=None, help="device to use, cuda for GPU / 計算を行うデバイス、cuda でGPUを使う")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+  svd(args)
diff --git a/networks/lora.py b/networks/lora.py
new file mode 100644
index 0000000000000000000000000000000000000000..5dc6b7d4455387a12ad318da252fd0974a5e3cf8
--- /dev/null
+++ b/networks/lora.py
@@ -0,0 +1,1221 @@
+# LoRA network module
+# reference:
+# https://github.com/microsoft/LoRA/blob/main/loralib/layers.py
+# https://github.com/cloneofsimo/lora/blob/master/lora_diffusion/lora.py
+
+import math
+import os
+from typing import Dict, List, Optional, Tuple, Type, Union
+from diffusers import AutoencoderKL
+from transformers import CLIPTextModel
+import numpy as np
+import torch
+import re
+
+
+RE_UPDOWN = re.compile(r"(up|down)_blocks_(\d+)_(resnets|upsamplers|downsamplers|attentions)_(\d+)_")
+
+RE_UPDOWN = re.compile(r"(up|down)_blocks_(\d+)_(resnets|upsamplers|downsamplers|attentions)_(\d+)_")
+
+
+class LoRAModule(torch.nn.Module):
+    """
+    replaces forward method of the original Linear, instead of replacing the original Linear module.
+    """
+
+    def __init__(
+        self,
+        lora_name,
+        org_module: torch.nn.Module,
+        multiplier=1.0,
+        lora_dim=4,
+        alpha=1,
+        dropout=None,
+        rank_dropout=None,
+        module_dropout=None,
+    ):
+        """if alpha == 0 or None, alpha is rank (no scaling)."""
+        super().__init__()
+        self.lora_name = lora_name
+
+        if org_module.__class__.__name__ == "Conv2d":
+            in_dim = org_module.in_channels
+            out_dim = org_module.out_channels
+        else:
+            in_dim = org_module.in_features
+            out_dim = org_module.out_features
+
+        # if limit_rank:
+        #   self.lora_dim = min(lora_dim, in_dim, out_dim)
+        #   if self.lora_dim != lora_dim:
+        #     print(f"{lora_name} dim (rank) is changed to: {self.lora_dim}")
+        # else:
+        self.lora_dim = lora_dim
+
+        if org_module.__class__.__name__ == "Conv2d":
+            kernel_size = org_module.kernel_size
+            stride = org_module.stride
+            padding = org_module.padding
+            self.lora_down = torch.nn.Conv2d(in_dim, self.lora_dim, kernel_size, stride, padding, bias=False)
+            self.lora_up = torch.nn.Conv2d(self.lora_dim, out_dim, (1, 1), (1, 1), bias=False)
+        else:
+            self.lora_down = torch.nn.Linear(in_dim, self.lora_dim, bias=False)
+            self.lora_up = torch.nn.Linear(self.lora_dim, out_dim, bias=False)
+
+        if type(alpha) == torch.Tensor:
+            alpha = alpha.detach().float().numpy()  # without casting, bf16 causes error
+        alpha = self.lora_dim if alpha is None or alpha == 0 else alpha
+        self.scale = alpha / self.lora_dim
+        self.register_buffer("alpha", torch.tensor(alpha))  # 定数として扱える
+
+        # same as microsoft's
+        torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5))
+        torch.nn.init.zeros_(self.lora_up.weight)
+
+        self.multiplier = multiplier
+        self.org_module = org_module  # remove in applying
+        self.dropout = dropout
+        self.rank_dropout = rank_dropout
+        self.module_dropout = module_dropout
+
+    def apply_to(self):
+        self.org_forward = self.org_module.forward
+        self.org_module.forward = self.forward
+        del self.org_module
+
+    def forward(self, x):
+        org_forwarded = self.org_forward(x)
+
+        # module dropout
+        if self.module_dropout is not None and self.training:
+            if torch.rand(1) < self.module_dropout:
+                return org_forwarded
+
+        lx = self.lora_down(x)
+
+        # normal dropout
+        if self.dropout is not None and self.training:
+            lx = torch.nn.functional.dropout(lx, p=self.dropout)
+
+        # rank dropout
+        if self.rank_dropout is not None and self.training:
+            mask = torch.rand((lx.size(0), self.lora_dim), device=lx.device) > self.rank_dropout
+            if len(lx.size()) == 3:
+                mask = mask.unsqueeze(1)  # for Text Encoder
+            elif len(lx.size()) == 4:
+                mask = mask.unsqueeze(-1).unsqueeze(-1)  # for Conv2d
+            lx = lx * mask
+
+            # scaling for rank dropout: treat as if the rank is changed
+            # maskから計算することも考えられるが、augmentation的な効果を期待してrank_dropoutを用いる
+            scale = self.scale * (1.0 / (1.0 - self.rank_dropout))  # redundant for readability
+        else:
+            scale = self.scale
+
+        lx = self.lora_up(lx)
+
+        return org_forwarded + lx * self.multiplier * scale
+
+
+class LoRAInfModule(LoRAModule):
+    def __init__(
+        self,
+        lora_name,
+        org_module: torch.nn.Module,
+        multiplier=1.0,
+        lora_dim=4,
+        alpha=1,
+        **kwargs,
+    ):
+        # no dropout for inference
+        super().__init__(lora_name, org_module, multiplier, lora_dim, alpha)
+
+        self.org_module_ref = [org_module]  # 後から参照できるように
+        self.enabled = True
+
+        # check regional or not by lora_name
+        self.text_encoder = False
+        if lora_name.startswith("lora_te_"):
+            self.regional = False
+            self.use_sub_prompt = True
+            self.text_encoder = True
+        elif "attn2_to_k" in lora_name or "attn2_to_v" in lora_name:
+            self.regional = False
+            self.use_sub_prompt = True
+        elif "time_emb" in lora_name:
+            self.regional = False
+            self.use_sub_prompt = False
+        else:
+            self.regional = True
+            self.use_sub_prompt = False
+
+        self.network: LoRANetwork = None
+
+    def set_network(self, network):
+        self.network = network
+
+    # freezeしてマージする
+    def merge_to(self, sd, dtype, device):
+        # get up/down weight
+        up_weight = sd["lora_up.weight"].to(torch.float).to(device)
+        down_weight = sd["lora_down.weight"].to(torch.float).to(device)
+
+        # extract weight from org_module
+        org_sd = self.org_module.state_dict()
+        weight = org_sd["weight"].to(torch.float)
+
+        # merge weight
+        if len(weight.size()) == 2:
+            # linear
+            weight = weight + self.multiplier * (up_weight @ down_weight) * self.scale
+        elif down_weight.size()[2:4] == (1, 1):
+            # conv2d 1x1
+            weight = (
+                weight
+                + self.multiplier
+                * (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3)
+                * self.scale
+            )
+        else:
+            # conv2d 3x3
+            conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3)
+            # print(conved.size(), weight.size(), module.stride, module.padding)
+            weight = weight + self.multiplier * conved * self.scale
+
+        # set weight to org_module
+        org_sd["weight"] = weight.to(dtype)
+        self.org_module.load_state_dict(org_sd)
+
+    # 復元できるマージのため、このモジュールのweightを返す
+    def get_weight(self, multiplier=None):
+        if multiplier is None:
+            multiplier = self.multiplier
+
+        # get up/down weight from module
+        up_weight = self.lora_up.weight.to(torch.float)
+        down_weight = self.lora_down.weight.to(torch.float)
+
+        # pre-calculated weight
+        if len(down_weight.size()) == 2:
+            # linear
+            weight = self.multiplier * (up_weight @ down_weight) * self.scale
+        elif down_weight.size()[2:4] == (1, 1):
+            # conv2d 1x1
+            weight = (
+                self.multiplier
+                * (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3)
+                * self.scale
+            )
+        else:
+            # conv2d 3x3
+            conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3)
+            weight = self.multiplier * conved * self.scale
+
+        return weight
+
+    def set_region(self, region):
+        self.region = region
+        self.region_mask = None
+
+    def default_forward(self, x):
+        # print("default_forward", self.lora_name, x.size())
+        return self.org_forward(x) + self.lora_up(self.lora_down(x)) * self.multiplier * self.scale
+
+    def forward(self, x):
+        if not self.enabled:
+            return self.org_forward(x)
+
+        if self.network is None or self.network.sub_prompt_index is None:
+            return self.default_forward(x)
+        if not self.regional and not self.use_sub_prompt:
+            return self.default_forward(x)
+
+        if self.regional:
+            return self.regional_forward(x)
+        else:
+            return self.sub_prompt_forward(x)
+
+    def get_mask_for_x(self, x):
+        # calculate size from shape of x
+        if len(x.size()) == 4:
+            h, w = x.size()[2:4]
+            area = h * w
+        else:
+            area = x.size()[1]
+
+        mask = self.network.mask_dic[area]
+        if mask is None:
+            raise ValueError(f"mask is None for resolution {area}")
+        if len(x.size()) != 4:
+            mask = torch.reshape(mask, (1, -1, 1))
+        return mask
+
+    def regional_forward(self, x):
+        if "attn2_to_out" in self.lora_name:
+            return self.to_out_forward(x)
+
+        if self.network.mask_dic is None:  # sub_prompt_index >= 3
+            return self.default_forward(x)
+
+        # apply mask for LoRA result
+        lx = self.lora_up(self.lora_down(x)) * self.multiplier * self.scale
+        mask = self.get_mask_for_x(lx)
+        # print("regional", self.lora_name, self.network.sub_prompt_index, lx.size(), mask.size())
+        lx = lx * mask
+
+        x = self.org_forward(x)
+        x = x + lx
+
+        if "attn2_to_q" in self.lora_name and self.network.is_last_network:
+            x = self.postp_to_q(x)
+
+        return x
+
+    def postp_to_q(self, x):
+        # repeat x to num_sub_prompts
+        has_real_uncond = x.size()[0] // self.network.batch_size == 3
+        qc = self.network.batch_size  # uncond
+        qc += self.network.batch_size * self.network.num_sub_prompts  # cond
+        if has_real_uncond:
+            qc += self.network.batch_size  # real_uncond
+
+        query = torch.zeros((qc, x.size()[1], x.size()[2]), device=x.device, dtype=x.dtype)
+        query[: self.network.batch_size] = x[: self.network.batch_size]
+
+        for i in range(self.network.batch_size):
+            qi = self.network.batch_size + i * self.network.num_sub_prompts
+            query[qi : qi + self.network.num_sub_prompts] = x[self.network.batch_size + i]
+
+        if has_real_uncond:
+            query[-self.network.batch_size :] = x[-self.network.batch_size :]
+
+        # print("postp_to_q", self.lora_name, x.size(), query.size(), self.network.num_sub_prompts)
+        return query
+
+    def sub_prompt_forward(self, x):
+        if x.size()[0] == self.network.batch_size:  # if uncond in text_encoder, do not apply LoRA
+            return self.org_forward(x)
+
+        emb_idx = self.network.sub_prompt_index
+        if not self.text_encoder:
+            emb_idx += self.network.batch_size
+
+        # apply sub prompt of X
+        lx = x[emb_idx :: self.network.num_sub_prompts]
+        lx = self.lora_up(self.lora_down(lx)) * self.multiplier * self.scale
+
+        # print("sub_prompt_forward", self.lora_name, x.size(), lx.size(), emb_idx)
+
+        x = self.org_forward(x)
+        x[emb_idx :: self.network.num_sub_prompts] += lx
+
+        return x
+
+    def to_out_forward(self, x):
+        # print("to_out_forward", self.lora_name, x.size(), self.network.is_last_network)
+
+        if self.network.is_last_network:
+            masks = [None] * self.network.num_sub_prompts
+            self.network.shared[self.lora_name] = (None, masks)
+        else:
+            lx, masks = self.network.shared[self.lora_name]
+
+        # call own LoRA
+        x1 = x[self.network.batch_size + self.network.sub_prompt_index :: self.network.num_sub_prompts]
+        lx1 = self.lora_up(self.lora_down(x1)) * self.multiplier * self.scale
+
+        if self.network.is_last_network:
+            lx = torch.zeros(
+                (self.network.num_sub_prompts * self.network.batch_size, *lx1.size()[1:]), device=lx1.device, dtype=lx1.dtype
+            )
+            self.network.shared[self.lora_name] = (lx, masks)
+
+        # print("to_out_forward", lx.size(), lx1.size(), self.network.sub_prompt_index, self.network.num_sub_prompts)
+        lx[self.network.sub_prompt_index :: self.network.num_sub_prompts] += lx1
+        masks[self.network.sub_prompt_index] = self.get_mask_for_x(lx1)
+
+        # if not last network, return x and masks
+        x = self.org_forward(x)
+        if not self.network.is_last_network:
+            return x
+
+        lx, masks = self.network.shared.pop(self.lora_name)
+
+        # if last network, combine separated x with mask weighted sum
+        has_real_uncond = x.size()[0] // self.network.batch_size == self.network.num_sub_prompts + 2
+
+        out = torch.zeros((self.network.batch_size * (3 if has_real_uncond else 2), *x.size()[1:]), device=x.device, dtype=x.dtype)
+        out[: self.network.batch_size] = x[: self.network.batch_size]  # uncond
+        if has_real_uncond:
+            out[-self.network.batch_size :] = x[-self.network.batch_size :]  # real_uncond
+
+        # print("to_out_forward", self.lora_name, self.network.sub_prompt_index, self.network.num_sub_prompts)
+        # for i in range(len(masks)):
+        #     if masks[i] is None:
+        #         masks[i] = torch.zeros_like(masks[-1])
+
+        mask = torch.cat(masks)
+        mask_sum = torch.sum(mask, dim=0) + 1e-4
+        for i in range(self.network.batch_size):
+            # 1枚の画像ごとに処理する
+            lx1 = lx[i * self.network.num_sub_prompts : (i + 1) * self.network.num_sub_prompts]
+            lx1 = lx1 * mask
+            lx1 = torch.sum(lx1, dim=0)
+
+            xi = self.network.batch_size + i * self.network.num_sub_prompts
+            x1 = x[xi : xi + self.network.num_sub_prompts]
+            x1 = x1 * mask
+            x1 = torch.sum(x1, dim=0)
+            x1 = x1 / mask_sum
+
+            x1 = x1 + lx1
+            out[self.network.batch_size + i] = x1
+
+        # print("to_out_forward", x.size(), out.size(), has_real_uncond)
+        return out
+
+
+def parse_block_lr_kwargs(nw_kwargs):
+    down_lr_weight = nw_kwargs.get("down_lr_weight", None)
+    mid_lr_weight = nw_kwargs.get("mid_lr_weight", None)
+    up_lr_weight = nw_kwargs.get("up_lr_weight", None)
+
+    # 以上のいずれにも設定がない場合は無効としてNoneを返す
+    if down_lr_weight is None and mid_lr_weight is None and up_lr_weight is None:
+        return None, None, None
+
+    # extract learning rate weight for each block
+    if down_lr_weight is not None:
+        # if some parameters are not set, use zero
+        if "," in down_lr_weight:
+            down_lr_weight = [(float(s) if s else 0.0) for s in down_lr_weight.split(",")]
+
+    if mid_lr_weight is not None:
+        mid_lr_weight = float(mid_lr_weight)
+
+    if up_lr_weight is not None:
+        if "," in up_lr_weight:
+            up_lr_weight = [(float(s) if s else 0.0) for s in up_lr_weight.split(",")]
+
+    down_lr_weight, mid_lr_weight, up_lr_weight = get_block_lr_weight(
+        down_lr_weight, mid_lr_weight, up_lr_weight, float(nw_kwargs.get("block_lr_zero_threshold", 0.0))
+    )
+
+    return down_lr_weight, mid_lr_weight, up_lr_weight
+
+
+def create_network(
+    multiplier: float,
+    network_dim: Optional[int],
+    network_alpha: Optional[float],
+    vae: AutoencoderKL,
+    text_encoder: Union[CLIPTextModel, List[CLIPTextModel]],
+    unet,
+    neuron_dropout: Optional[float] = None,
+    **kwargs,
+):
+    if network_dim is None:
+        network_dim = 4  # default
+    if network_alpha is None:
+        network_alpha = 1.0
+
+    # extract dim/alpha for conv2d, and block dim
+    conv_dim = kwargs.get("conv_dim", None)
+    conv_alpha = kwargs.get("conv_alpha", None)
+    if conv_dim is not None:
+        conv_dim = int(conv_dim)
+        if conv_alpha is None:
+            conv_alpha = 1.0
+        else:
+            conv_alpha = float(conv_alpha)
+
+    # block dim/alpha/lr
+    block_dims = kwargs.get("block_dims", None)
+    down_lr_weight, mid_lr_weight, up_lr_weight = parse_block_lr_kwargs(kwargs)
+
+    # 以上のいずれかに指定があればblockごとのdim(rank)を有効にする
+    if block_dims is not None or down_lr_weight is not None or mid_lr_weight is not None or up_lr_weight is not None:
+        block_alphas = kwargs.get("block_alphas", None)
+        conv_block_dims = kwargs.get("conv_block_dims", None)
+        conv_block_alphas = kwargs.get("conv_block_alphas", None)
+
+        block_dims, block_alphas, conv_block_dims, conv_block_alphas = get_block_dims_and_alphas(
+            block_dims, block_alphas, network_dim, network_alpha, conv_block_dims, conv_block_alphas, conv_dim, conv_alpha
+        )
+
+        # remove block dim/alpha without learning rate
+        block_dims, block_alphas, conv_block_dims, conv_block_alphas = remove_block_dims_and_alphas(
+            block_dims, block_alphas, conv_block_dims, conv_block_alphas, down_lr_weight, mid_lr_weight, up_lr_weight
+        )
+
+    else:
+        block_alphas = None
+        conv_block_dims = None
+        conv_block_alphas = None
+
+    # rank/module dropout
+    rank_dropout = kwargs.get("rank_dropout", None)
+    if rank_dropout is not None:
+        rank_dropout = float(rank_dropout)
+    module_dropout = kwargs.get("module_dropout", None)
+    if module_dropout is not None:
+        module_dropout = float(module_dropout)
+
+    # すごく引数が多いな ( ^ω^)･･･
+    network = LoRANetwork(
+        text_encoder,
+        unet,
+        multiplier=multiplier,
+        lora_dim=network_dim,
+        alpha=network_alpha,
+        dropout=neuron_dropout,
+        rank_dropout=rank_dropout,
+        module_dropout=module_dropout,
+        conv_lora_dim=conv_dim,
+        conv_alpha=conv_alpha,
+        block_dims=block_dims,
+        block_alphas=block_alphas,
+        conv_block_dims=conv_block_dims,
+        conv_block_alphas=conv_block_alphas,
+        varbose=True,
+    )
+
+    if up_lr_weight is not None or mid_lr_weight is not None or down_lr_weight is not None:
+        network.set_block_lr_weight(up_lr_weight, mid_lr_weight, down_lr_weight)
+
+    return network
+
+
+# このメソッドは外部から呼び出される可能性を考慮しておく
+# network_dim, network_alpha にはデフォルト値が入っている。
+# block_dims, block_alphas は両方ともNoneまたは両方とも値が入っている
+# conv_dim, conv_alpha は両方ともNoneまたは両方とも値が入っている
+def get_block_dims_and_alphas(
+    block_dims, block_alphas, network_dim, network_alpha, conv_block_dims, conv_block_alphas, conv_dim, conv_alpha
+):
+    num_total_blocks = LoRANetwork.NUM_OF_BLOCKS * 2 + 1
+
+    def parse_ints(s):
+        return [int(i) for i in s.split(",")]
+
+    def parse_floats(s):
+        return [float(i) for i in s.split(",")]
+
+    # block_dimsとblock_alphasをパースする。必ず値が入る
+    if block_dims is not None:
+        block_dims = parse_ints(block_dims)
+        assert (
+            len(block_dims) == num_total_blocks
+        ), f"block_dims must have {num_total_blocks} elements / block_dimsは{num_total_blocks}個指定してください"
+    else:
+        print(f"block_dims is not specified. all dims are set to {network_dim} / block_dimsが指定されていません。すべてのdimは{network_dim}になります")
+        block_dims = [network_dim] * num_total_blocks
+
+    if block_alphas is not None:
+        block_alphas = parse_floats(block_alphas)
+        assert (
+            len(block_alphas) == num_total_blocks
+        ), f"block_alphas must have {num_total_blocks} elements / block_alphasは{num_total_blocks}個指定してください"
+    else:
+        print(
+            f"block_alphas is not specified. all alphas are set to {network_alpha} / block_alphasが指定されていません。すべてのalphaは{network_alpha}になります"
+        )
+        block_alphas = [network_alpha] * num_total_blocks
+
+    # conv_block_dimsとconv_block_alphasを、指定がある場合のみパースする。指定がなければconv_dimとconv_alphaを使う
+    if conv_block_dims is not None:
+        conv_block_dims = parse_ints(conv_block_dims)
+        assert (
+            len(conv_block_dims) == num_total_blocks
+        ), f"conv_block_dims must have {num_total_blocks} elements / conv_block_dimsは{num_total_blocks}個指定してください"
+
+        if conv_block_alphas is not None:
+            conv_block_alphas = parse_floats(conv_block_alphas)
+            assert (
+                len(conv_block_alphas) == num_total_blocks
+            ), f"conv_block_alphas must have {num_total_blocks} elements / conv_block_alphasは{num_total_blocks}個指定してください"
+        else:
+            if conv_alpha is None:
+                conv_alpha = 1.0
+            print(
+                f"conv_block_alphas is not specified. all alphas are set to {conv_alpha} / conv_block_alphasが指定されていません。すべてのalphaは{conv_alpha}になります"
+            )
+            conv_block_alphas = [conv_alpha] * num_total_blocks
+    else:
+        if conv_dim is not None:
+            print(
+                f"conv_dim/alpha for all blocks are set to {conv_dim} and {conv_alpha} / すべてのブロックのconv_dimとalphaは{conv_dim}および{conv_alpha}になります"
+            )
+            conv_block_dims = [conv_dim] * num_total_blocks
+            conv_block_alphas = [conv_alpha] * num_total_blocks
+        else:
+            conv_block_dims = None
+            conv_block_alphas = None
+
+    return block_dims, block_alphas, conv_block_dims, conv_block_alphas
+
+
+# 層別学習率用に層ごとの学習率に対する倍率を定義する、外部から呼び出される可能性を考慮しておく
+def get_block_lr_weight(
+    down_lr_weight, mid_lr_weight, up_lr_weight, zero_threshold
+) -> Tuple[List[float], List[float], List[float]]:
+    # パラメータ未指定時は何もせず、今までと同じ動作とする
+    if up_lr_weight is None and mid_lr_weight is None and down_lr_weight is None:
+        return None, None, None
+
+    max_len = LoRANetwork.NUM_OF_BLOCKS  # フルモデル相当でのup,downの層の数
+
+    def get_list(name_with_suffix) -> List[float]:
+        import math
+
+        tokens = name_with_suffix.split("+")
+        name = tokens[0]
+        base_lr = float(tokens[1]) if len(tokens) > 1 else 0.0
+
+        if name == "cosine":
+            return [math.sin(math.pi * (i / (max_len - 1)) / 2) + base_lr for i in reversed(range(max_len))]
+        elif name == "sine":
+            return [math.sin(math.pi * (i / (max_len - 1)) / 2) + base_lr for i in range(max_len)]
+        elif name == "linear":
+            return [i / (max_len - 1) + base_lr for i in range(max_len)]
+        elif name == "reverse_linear":
+            return [i / (max_len - 1) + base_lr for i in reversed(range(max_len))]
+        elif name == "zeros":
+            return [0.0 + base_lr] * max_len
+        else:
+            print(
+                "Unknown lr_weight argument %s is used. Valid arguments:  / 不明なlr_weightの引数 %s が使われました。有効な引数:\n\tcosine, sine, linear, reverse_linear, zeros"
+                % (name)
+            )
+            return None
+
+    if type(down_lr_weight) == str:
+        down_lr_weight = get_list(down_lr_weight)
+    if type(up_lr_weight) == str:
+        up_lr_weight = get_list(up_lr_weight)
+
+    if (up_lr_weight != None and len(up_lr_weight) > max_len) or (down_lr_weight != None and len(down_lr_weight) > max_len):
+        print("down_weight or up_weight is too long. Parameters after %d-th are ignored." % max_len)
+        print("down_weightもしくはup_weightが長すぎます。%d個目以降のパラメータは無視されます。" % max_len)
+        up_lr_weight = up_lr_weight[:max_len]
+        down_lr_weight = down_lr_weight[:max_len]
+
+    if (up_lr_weight != None and len(up_lr_weight) < max_len) or (down_lr_weight != None and len(down_lr_weight) < max_len):
+        print("down_weight or up_weight is too short. Parameters after %d-th are filled with 1." % max_len)
+        print("down_weightもしくはup_weightが短すぎます。%d個目までの不足したパラメータは1で補われます。" % max_len)
+
+        if down_lr_weight != None and len(down_lr_weight) < max_len:
+            down_lr_weight = down_lr_weight + [1.0] * (max_len - len(down_lr_weight))
+        if up_lr_weight != None and len(up_lr_weight) < max_len:
+            up_lr_weight = up_lr_weight + [1.0] * (max_len - len(up_lr_weight))
+
+    if (up_lr_weight != None) or (mid_lr_weight != None) or (down_lr_weight != None):
+        print("apply block learning rate / 階層別学習率を適用します。")
+        if down_lr_weight != None:
+            down_lr_weight = [w if w > zero_threshold else 0 for w in down_lr_weight]
+            print("down_lr_weight (shallower -> deeper, 浅い層->深い層):", down_lr_weight)
+        else:
+            print("down_lr_weight: all 1.0, すべて1.0")
+
+        if mid_lr_weight != None:
+            mid_lr_weight = mid_lr_weight if mid_lr_weight > zero_threshold else 0
+            print("mid_lr_weight:", mid_lr_weight)
+        else:
+            print("mid_lr_weight: 1.0")
+
+        if up_lr_weight != None:
+            up_lr_weight = [w if w > zero_threshold else 0 for w in up_lr_weight]
+            print("up_lr_weight (deeper -> shallower, 深い層->浅い層):", up_lr_weight)
+        else:
+            print("up_lr_weight: all 1.0, すべて1.0")
+
+    return down_lr_weight, mid_lr_weight, up_lr_weight
+
+
+# lr_weightが0のblockをblock_dimsから除外する、外部から呼び出す可能性を考慮しておく
+def remove_block_dims_and_alphas(
+    block_dims, block_alphas, conv_block_dims, conv_block_alphas, down_lr_weight, mid_lr_weight, up_lr_weight
+):
+    # set 0 to block dim without learning rate to remove the block
+    if down_lr_weight != None:
+        for i, lr in enumerate(down_lr_weight):
+            if lr == 0:
+                block_dims[i] = 0
+                if conv_block_dims is not None:
+                    conv_block_dims[i] = 0
+    if mid_lr_weight != None:
+        if mid_lr_weight == 0:
+            block_dims[LoRANetwork.NUM_OF_BLOCKS] = 0
+            if conv_block_dims is not None:
+                conv_block_dims[LoRANetwork.NUM_OF_BLOCKS] = 0
+    if up_lr_weight != None:
+        for i, lr in enumerate(up_lr_weight):
+            if lr == 0:
+                block_dims[LoRANetwork.NUM_OF_BLOCKS + 1 + i] = 0
+                if conv_block_dims is not None:
+                    conv_block_dims[LoRANetwork.NUM_OF_BLOCKS + 1 + i] = 0
+
+    return block_dims, block_alphas, conv_block_dims, conv_block_alphas
+
+
+# 外部から呼び出す可能性を考慮しておく
+def get_block_index(lora_name: str) -> int:
+    block_idx = -1  # invalid lora name
+
+    m = RE_UPDOWN.search(lora_name)
+    if m:
+        g = m.groups()
+        i = int(g[1])
+        j = int(g[3])
+        if g[2] == "resnets":
+            idx = 3 * i + j
+        elif g[2] == "attentions":
+            idx = 3 * i + j
+        elif g[2] == "upsamplers" or g[2] == "downsamplers":
+            idx = 3 * i + 2
+
+        if g[0] == "down":
+            block_idx = 1 + idx  # 0に該当するLoRAは存在しない
+        elif g[0] == "up":
+            block_idx = LoRANetwork.NUM_OF_BLOCKS + 1 + idx
+
+    elif "mid_block_" in lora_name:
+        block_idx = LoRANetwork.NUM_OF_BLOCKS  # idx=12
+
+    return block_idx
+
+
+# Create network from weights for inference, weights are not loaded here (because can be merged)
+def create_network_from_weights(multiplier, file, vae, text_encoder, unet, weights_sd=None, for_inference=False, **kwargs):
+    if weights_sd is None:
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import load_file, safe_open
+
+            weights_sd = load_file(file)
+        else:
+            weights_sd = torch.load(file, map_location="cpu")
+
+    # get dim/alpha mapping
+    modules_dim = {}
+    modules_alpha = {}
+    for key, value in weights_sd.items():
+        if "." not in key:
+            continue
+
+        lora_name = key.split(".")[0]
+        if "alpha" in key:
+            modules_alpha[lora_name] = value
+        elif "lora_down" in key:
+            dim = value.size()[0]
+            modules_dim[lora_name] = dim
+            # print(lora_name, value.size(), dim)
+
+    # support old LoRA without alpha
+    for key in modules_dim.keys():
+        if key not in modules_alpha:
+            modules_alpha[key] = modules_dim[key]
+
+    module_class = LoRAInfModule if for_inference else LoRAModule
+
+    network = LoRANetwork(
+        text_encoder, unet, multiplier=multiplier, modules_dim=modules_dim, modules_alpha=modules_alpha, module_class=module_class
+    )
+
+    # block lr
+    down_lr_weight, mid_lr_weight, up_lr_weight = parse_block_lr_kwargs(kwargs)
+    if up_lr_weight is not None or mid_lr_weight is not None or down_lr_weight is not None:
+        network.set_block_lr_weight(up_lr_weight, mid_lr_weight, down_lr_weight)
+
+    return network, weights_sd
+
+
+class LoRANetwork(torch.nn.Module):
+    NUM_OF_BLOCKS = 12  # フルモデル相当でのup,downの層の数
+
+    UNET_TARGET_REPLACE_MODULE = ["Transformer2DModel"]
+    UNET_TARGET_REPLACE_MODULE_CONV2D_3X3 = ["ResnetBlock2D", "Downsample2D", "Upsample2D"]
+    TEXT_ENCODER_TARGET_REPLACE_MODULE = ["CLIPAttention", "CLIPMLP"]
+    LORA_PREFIX_UNET = "lora_unet"
+    LORA_PREFIX_TEXT_ENCODER = "lora_te"
+
+    # SDXL: must starts with LORA_PREFIX_TEXT_ENCODER
+    LORA_PREFIX_TEXT_ENCODER1 = "lora_te1"
+    LORA_PREFIX_TEXT_ENCODER2 = "lora_te2"
+
+    def __init__(
+        self,
+        text_encoder: Union[List[CLIPTextModel], CLIPTextModel],
+        unet,
+        multiplier: float = 1.0,
+        lora_dim: int = 4,
+        alpha: float = 1,
+        dropout: Optional[float] = None,
+        rank_dropout: Optional[float] = None,
+        module_dropout: Optional[float] = None,
+        conv_lora_dim: Optional[int] = None,
+        conv_alpha: Optional[float] = None,
+        block_dims: Optional[List[int]] = None,
+        block_alphas: Optional[List[float]] = None,
+        conv_block_dims: Optional[List[int]] = None,
+        conv_block_alphas: Optional[List[float]] = None,
+        modules_dim: Optional[Dict[str, int]] = None,
+        modules_alpha: Optional[Dict[str, int]] = None,
+        module_class: Type[object] = LoRAModule,
+        varbose: Optional[bool] = False,
+    ) -> None:
+        """
+        LoRA network: すごく引数が多いが、パターンは以下の通り
+        1. lora_dimとalphaを指定
+        2. lora_dim、alpha、conv_lora_dim、conv_alphaを指定
+        3. block_dimsとblock_alphasを指定 :  Conv2d3x3には適用しない
+        4. block_dims、block_alphas、conv_block_dims、conv_block_alphasを指定 : Conv2d3x3にも適用する
+        5. modules_dimとmodules_alphaを指定 (推論用)
+        """
+        super().__init__()
+        self.multiplier = multiplier
+
+        self.lora_dim = lora_dim
+        self.alpha = alpha
+        self.conv_lora_dim = conv_lora_dim
+        self.conv_alpha = conv_alpha
+        self.dropout = dropout
+        self.rank_dropout = rank_dropout
+        self.module_dropout = module_dropout
+
+        if modules_dim is not None:
+            print(f"create LoRA network from weights")
+        elif block_dims is not None:
+            print(f"create LoRA network from block_dims")
+            print(f"neuron dropout: p={self.dropout}, rank dropout: p={self.rank_dropout}, module dropout: p={self.module_dropout}")
+            print(f"block_dims: {block_dims}")
+            print(f"block_alphas: {block_alphas}")
+            if conv_block_dims is not None:
+                print(f"conv_block_dims: {conv_block_dims}")
+                print(f"conv_block_alphas: {conv_block_alphas}")
+        else:
+            print(f"create LoRA network. base dim (rank): {lora_dim}, alpha: {alpha}")
+            print(f"neuron dropout: p={self.dropout}, rank dropout: p={self.rank_dropout}, module dropout: p={self.module_dropout}")
+            if self.conv_lora_dim is not None:
+                print(f"apply LoRA to Conv2d with kernel size (3,3). dim (rank): {self.conv_lora_dim}, alpha: {self.conv_alpha}")
+
+        # create module instances
+        def create_modules(
+            is_unet: bool,
+            text_encoder_idx: Optional[int],  # None, 1, 2
+            root_module: torch.nn.Module,
+            target_replace_modules: List[torch.nn.Module],
+        ) -> List[LoRAModule]:
+            prefix = (
+                self.LORA_PREFIX_UNET
+                if is_unet
+                else (
+                    self.LORA_PREFIX_TEXT_ENCODER
+                    if text_encoder_idx is None
+                    else (self.LORA_PREFIX_TEXT_ENCODER1 if text_encoder_idx == 1 else self.LORA_PREFIX_TEXT_ENCODER2)
+                )
+            )
+            loras = []
+            skipped = []
+            for name, module in root_module.named_modules():
+                if module.__class__.__name__ in target_replace_modules:
+                    for child_name, child_module in module.named_modules():
+                        is_linear = child_module.__class__.__name__ == "Linear"
+                        is_conv2d = child_module.__class__.__name__ == "Conv2d"
+                        is_conv2d_1x1 = is_conv2d and child_module.kernel_size == (1, 1)
+
+                        if is_linear or is_conv2d:
+                            lora_name = prefix + "." + name + "." + child_name
+                            lora_name = lora_name.replace(".", "_")
+
+                            dim = None
+                            alpha = None
+
+                            if modules_dim is not None:
+                                # モジュール指定あり
+                                if lora_name in modules_dim:
+                                    dim = modules_dim[lora_name]
+                                    alpha = modules_alpha[lora_name]
+                            elif is_unet and block_dims is not None:
+                                # U-Netでblock_dims指定あり
+                                block_idx = get_block_index(lora_name)
+                                if is_linear or is_conv2d_1x1:
+                                    dim = block_dims[block_idx]
+                                    alpha = block_alphas[block_idx]
+                                elif conv_block_dims is not None:
+                                    dim = conv_block_dims[block_idx]
+                                    alpha = conv_block_alphas[block_idx]
+                            else:
+                                # 通常、すべて対象とする
+                                if is_linear or is_conv2d_1x1:
+                                    dim = self.lora_dim
+                                    alpha = self.alpha
+                                elif self.conv_lora_dim is not None:
+                                    dim = self.conv_lora_dim
+                                    alpha = self.conv_alpha
+
+                            if dim is None or dim == 0:
+                                # skipした情報を出力
+                                if is_linear or is_conv2d_1x1 or (self.conv_lora_dim is not None or conv_block_dims is not None):
+                                    skipped.append(lora_name)
+                                continue
+
+                            lora = module_class(
+                                lora_name,
+                                child_module,
+                                self.multiplier,
+                                dim,
+                                alpha,
+                                dropout=dropout,
+                                rank_dropout=rank_dropout,
+                                module_dropout=module_dropout,
+                            )
+                            loras.append(lora)
+            return loras, skipped
+
+        text_encoders = text_encoder if type(text_encoder) == list else [text_encoder]
+
+        # create LoRA for text encoder
+        # 毎回すべてのモジュールを作るのは無駄なので要検討
+        self.text_encoder_loras = []
+        skipped_te = []
+        for i, text_encoder in enumerate(text_encoders):
+            if len(text_encoders) > 1:
+                index = i + 1
+                print(f"create LoRA for Text Encoder {index}:")
+            else:
+                index = None
+                print(f"create LoRA for Text Encoder:")
+
+            text_encoder_loras, skipped = create_modules(False, index, text_encoder, LoRANetwork.TEXT_ENCODER_TARGET_REPLACE_MODULE)
+            self.text_encoder_loras.extend(text_encoder_loras)
+            skipped_te += skipped
+        print(f"create LoRA for Text Encoder: {len(self.text_encoder_loras)} modules.")
+
+        # extend U-Net target modules if conv2d 3x3 is enabled, or load from weights
+        target_modules = LoRANetwork.UNET_TARGET_REPLACE_MODULE
+        if modules_dim is not None or self.conv_lora_dim is not None or conv_block_dims is not None:
+            target_modules += LoRANetwork.UNET_TARGET_REPLACE_MODULE_CONV2D_3X3
+
+        self.unet_loras, skipped_un = create_modules(True, None, unet, target_modules)
+        print(f"create LoRA for U-Net: {len(self.unet_loras)} modules.")
+
+        skipped = skipped_te + skipped_un
+        if varbose and len(skipped) > 0:
+            print(
+                f"because block_lr_weight is 0 or dim (rank) is 0, {len(skipped)} LoRA modules are skipped / block_lr_weightまたはdim (rank)が0の為、次の{len(skipped)}個のLoRAモジュールはスキップされます:"
+            )
+            for name in skipped:
+                print(f"\t{name}")
+
+        self.up_lr_weight: List[float] = None
+        self.down_lr_weight: List[float] = None
+        self.mid_lr_weight: float = None
+        self.block_lr = False
+
+        # assertion
+        names = set()
+        for lora in self.text_encoder_loras + self.unet_loras:
+            assert lora.lora_name not in names, f"duplicated lora name: {lora.lora_name}"
+            names.add(lora.lora_name)
+
+    def set_multiplier(self, multiplier):
+        self.multiplier = multiplier
+        for lora in self.text_encoder_loras + self.unet_loras:
+            lora.multiplier = self.multiplier
+
+    def load_weights(self, file):
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import load_file
+
+            weights_sd = load_file(file)
+        else:
+            weights_sd = torch.load(file, map_location="cpu")
+        info = self.load_state_dict(weights_sd, False)
+        return info
+
+    def apply_to(self, text_encoder, unet, apply_text_encoder=True, apply_unet=True):
+        if apply_text_encoder:
+            print("enable LoRA for text encoder")
+        else:
+            self.text_encoder_loras = []
+
+        if apply_unet:
+            print("enable LoRA for U-Net")
+        else:
+            self.unet_loras = []
+
+        for lora in self.text_encoder_loras + self.unet_loras:
+            lora.apply_to()
+            self.add_module(lora.lora_name, lora)
+
+    # マージできるかどうかを返す
+    def is_mergeable(self):
+        return True
+
+    # TODO refactor to common function with apply_to
+    def merge_to(self, text_encoder, unet, weights_sd, dtype, device):
+        apply_text_encoder = apply_unet = False
+        for key in weights_sd.keys():
+            if key.startswith(LoRANetwork.LORA_PREFIX_TEXT_ENCODER):
+                apply_text_encoder = True
+            elif key.startswith(LoRANetwork.LORA_PREFIX_UNET):
+                apply_unet = True
+
+        if apply_text_encoder:
+            print("enable LoRA for text encoder")
+        else:
+            self.text_encoder_loras = []
+
+        if apply_unet:
+            print("enable LoRA for U-Net")
+        else:
+            self.unet_loras = []
+
+        for lora in self.text_encoder_loras + self.unet_loras:
+            sd_for_lora = {}
+            for key in weights_sd.keys():
+                if key.startswith(lora.lora_name):
+                    sd_for_lora[key[len(lora.lora_name) + 1 :]] = weights_sd[key]
+            lora.merge_to(sd_for_lora, dtype, device)
+
+        print(f"weights are merged")
+
+    # 層別学習率用に層ごとの学習率に対する倍率を定義する　引数の順番が逆だがとりあえず気にしない
+    def set_block_lr_weight(
+        self,
+        up_lr_weight: List[float] = None,
+        mid_lr_weight: float = None,
+        down_lr_weight: List[float] = None,
+    ):
+        self.block_lr = True
+        self.down_lr_weight = down_lr_weight
+        self.mid_lr_weight = mid_lr_weight
+        self.up_lr_weight = up_lr_weight
+
+    def get_lr_weight(self, lora: LoRAModule) -> float:
+        lr_weight = 1.0
+        block_idx = get_block_index(lora.lora_name)
+        if block_idx < 0:
+            return lr_weight
+
+        if block_idx < LoRANetwork.NUM_OF_BLOCKS:
+            if self.down_lr_weight != None:
+                lr_weight = self.down_lr_weight[block_idx]
+        elif block_idx == LoRANetwork.NUM_OF_BLOCKS:
+            if self.mid_lr_weight != None:
+                lr_weight = self.mid_lr_weight
+        elif block_idx > LoRANetwork.NUM_OF_BLOCKS:
+            if self.up_lr_weight != None:
+                lr_weight = self.up_lr_weight[block_idx - LoRANetwork.NUM_OF_BLOCKS - 1]
+
+        return lr_weight
+
+    # 二つのText Encoderに別々の学習率を設定できるようにするといいかも
+    def prepare_optimizer_params(self, text_encoder_lr, unet_lr, default_lr):
+        self.requires_grad_(True)
+        all_params = []
+
+        def enumerate_params(loras):
+            params = []
+            for lora in loras:
+                params.extend(lora.parameters())
+            return params
+
+        if self.text_encoder_loras:
+            param_data = {"params": enumerate_params(self.text_encoder_loras)}
+            if text_encoder_lr is not None:
+                param_data["lr"] = text_encoder_lr
+            all_params.append(param_data)
+
+        if self.unet_loras:
+            if self.block_lr:
+                # 学習率のグラフをblockごとにしたいので、blockごとにloraを分類
+                block_idx_to_lora = {}
+                for lora in self.unet_loras:
+                    idx = get_block_index(lora.lora_name)
+                    if idx not in block_idx_to_lora:
+                        block_idx_to_lora[idx] = []
+                    block_idx_to_lora[idx].append(lora)
+
+                # blockごとにパラメータを設定する
+                for idx, block_loras in block_idx_to_lora.items():
+                    param_data = {"params": enumerate_params(block_loras)}
+
+                    if unet_lr is not None:
+                        param_data["lr"] = unet_lr * self.get_lr_weight(block_loras[0])
+                    elif default_lr is not None:
+                        param_data["lr"] = default_lr * self.get_lr_weight(block_loras[0])
+                    if ("lr" in param_data) and (param_data["lr"] == 0):
+                        continue
+                    all_params.append(param_data)
+
+            else:
+                param_data = {"params": enumerate_params(self.unet_loras)}
+                if unet_lr is not None:
+                    param_data["lr"] = unet_lr
+                all_params.append(param_data)
+
+        return all_params
+
+    def enable_gradient_checkpointing(self):
+        # not supported
+        pass
+
+    def prepare_grad_etc(self, text_encoder, unet):
+        self.requires_grad_(True)
+
+    def on_epoch_start(self, text_encoder, unet):
+        self.train()
+
+    def get_trainable_params(self):
+        return self.parameters()
+
+    def save_weights(self, file, dtype, metadata):
+        if metadata is not None and len(metadata) == 0:
+            metadata = None
+
+        state_dict = self.state_dict()
+
+        if dtype is not None:
+            for key in list(state_dict.keys()):
+                v = state_dict[key]
+                v = v.detach().clone().to("cpu").to(dtype)
+                state_dict[key] = v
+
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import save_file
+            from library import train_util
+
+            # Precalculate model hashes to save time on indexing
+            if metadata is None:
+                metadata = {}
+            model_hash, legacy_hash = train_util.precalculate_safetensors_hashes(state_dict, metadata)
+            metadata["sshs_model_hash"] = model_hash
+            metadata["sshs_legacy_hash"] = legacy_hash
+
+            save_file(state_dict, file, metadata)
+        else:
+            torch.save(state_dict, file)
+
+    # mask is a tensor with values from 0 to 1
+    def set_region(self, sub_prompt_index, is_last_network, mask):
+        if mask.max() == 0:
+            mask = torch.ones_like(mask)
+
+        self.mask = mask
+        self.sub_prompt_index = sub_prompt_index
+        self.is_last_network = is_last_network
+
+        for lora in self.text_encoder_loras + self.unet_loras:
+            lora.set_network(self)
+
+    def set_current_generation(self, batch_size, num_sub_prompts, width, height, shared):
+        self.batch_size = batch_size
+        self.num_sub_prompts = num_sub_prompts
+        self.current_size = (height, width)
+        self.shared = shared
+
+        # create masks
+        mask = self.mask
+        mask_dic = {}
+        mask = mask.unsqueeze(0).unsqueeze(1)  # b(1),c(1),h,w
+        ref_weight = self.text_encoder_loras[0].lora_down.weight if self.text_encoder_loras else self.unet_loras[0].lora_down.weight
+        dtype = ref_weight.dtype
+        device = ref_weight.device
+
+        def resize_add(mh, mw):
+            # print(mh, mw, mh * mw)
+            m = torch.nn.functional.interpolate(mask, (mh, mw), mode="bilinear")  # doesn't work in bf16
+            m = m.to(device, dtype=dtype)
+            mask_dic[mh * mw] = m
+
+        h = height // 8
+        w = width // 8
+        for _ in range(4):
+            resize_add(h, w)
+            if h % 2 == 1 or w % 2 == 1:  # add extra shape if h/w is not divisible by 2
+                resize_add(h + h % 2, w + w % 2)
+            h = (h + 1) // 2
+            w = (w + 1) // 2
+
+        self.mask_dic = mask_dic
+
+    def backup_weights(self):
+        # 重みのバックアップを行う
+        loras: List[LoRAInfModule] = self.text_encoder_loras + self.unet_loras
+        for lora in loras:
+            org_module = lora.org_module_ref[0]
+            if not hasattr(org_module, "_lora_org_weight"):
+                sd = org_module.state_dict()
+                org_module._lora_org_weight = sd["weight"].detach().clone()
+                org_module._lora_restored = True
+
+    def restore_weights(self):
+        # 重みのリストアを行う
+        loras: List[LoRAInfModule] = self.text_encoder_loras + self.unet_loras
+        for lora in loras:
+            org_module = lora.org_module_ref[0]
+            if not org_module._lora_restored:
+                sd = org_module.state_dict()
+                sd["weight"] = org_module._lora_org_weight
+                org_module.load_state_dict(sd)
+                org_module._lora_restored = True
+
+    def pre_calculation(self):
+        # 事前計算を行う
+        loras: List[LoRAInfModule] = self.text_encoder_loras + self.unet_loras
+        for lora in loras:
+            org_module = lora.org_module_ref[0]
+            sd = org_module.state_dict()
+
+            org_weight = sd["weight"]
+            lora_weight = lora.get_weight().to(org_weight.device, dtype=org_weight.dtype)
+            sd["weight"] = org_weight + lora_weight
+            assert sd["weight"].shape == org_weight.shape
+            org_module.load_state_dict(sd)
+
+            org_module._lora_restored = False
+            lora.enabled = False
+
+    def apply_max_norm_regularization(self, max_norm_value, device):
+        downkeys = []
+        upkeys = []
+        alphakeys = []
+        norms = []
+        keys_scaled = 0
+
+        state_dict = self.state_dict()
+        for key in state_dict.keys():
+            if "lora_down" in key and "weight" in key:
+                downkeys.append(key)
+                upkeys.append(key.replace("lora_down", "lora_up"))
+                alphakeys.append(key.replace("lora_down.weight", "alpha"))
+
+        for i in range(len(downkeys)):
+            down = state_dict[downkeys[i]].to(device)
+            up = state_dict[upkeys[i]].to(device)
+            alpha = state_dict[alphakeys[i]].to(device)
+            dim = down.shape[0]
+            scale = alpha / dim
+
+            if up.shape[2:] == (1, 1) and down.shape[2:] == (1, 1):
+                updown = (up.squeeze(2).squeeze(2) @ down.squeeze(2).squeeze(2)).unsqueeze(2).unsqueeze(3)
+            elif up.shape[2:] == (3, 3) or down.shape[2:] == (3, 3):
+                updown = torch.nn.functional.conv2d(down.permute(1, 0, 2, 3), up).permute(1, 0, 2, 3)
+            else:
+                updown = up @ down
+
+            updown *= scale
+
+            norm = updown.norm().clamp(min=max_norm_value / 2)
+            desired = torch.clamp(norm, max=max_norm_value)
+            ratio = desired.cpu() / norm.cpu()
+            sqrt_ratio = ratio**0.5
+            if ratio != 1:
+                keys_scaled += 1
+                state_dict[upkeys[i]] *= sqrt_ratio
+                state_dict[downkeys[i]] *= sqrt_ratio
+            scalednorm = updown.norm() * ratio
+            norms.append(scalednorm.item())
+
+        return keys_scaled, sum(norms) / len(norms), max(norms)
diff --git a/networks/lora_interrogator.py b/networks/lora_interrogator.py
new file mode 100644
index 0000000000000000000000000000000000000000..0dc066fd1525d873e280c970707d24d0061db9e2
--- /dev/null
+++ b/networks/lora_interrogator.py
@@ -0,0 +1,139 @@
+
+
+from tqdm import tqdm
+from library import model_util
+import library.train_util as train_util
+import argparse
+from transformers import CLIPTokenizer
+import torch
+
+import library.model_util as model_util
+import lora
+
+TOKENIZER_PATH = "openai/clip-vit-large-patch14"
+V2_STABLE_DIFFUSION_PATH = "stabilityai/stable-diffusion-2"     # ここからtokenizerだけ使う
+
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+
+def interrogate(args):
+  weights_dtype = torch.float16
+
+  # いろいろ準備する
+  print(f"loading SD model: {args.sd_model}")
+  args.pretrained_model_name_or_path = args.sd_model
+  args.vae = None
+  text_encoder, vae, unet, _ = train_util._load_target_model(args,weights_dtype, DEVICE)
+
+  print(f"loading LoRA: {args.model}")
+  network, weights_sd = lora.create_network_from_weights(1.0, args.model, vae, text_encoder, unet)
+
+  # text encoder向けの重みがあるかチェックする：本当はlora側でやるのがいい
+  has_te_weight = False
+  for key in weights_sd.keys():
+    if 'lora_te' in key:
+      has_te_weight = True
+      break
+  if not has_te_weight:
+    print("This LoRA does not have modules for Text Encoder, cannot interrogate / このLoRAはText Encoder向けのモジュールがないため調査できません")
+    return
+  del vae
+
+  print("loading tokenizer")
+  if args.v2:
+    tokenizer: CLIPTokenizer = CLIPTokenizer.from_pretrained(V2_STABLE_DIFFUSION_PATH, subfolder="tokenizer")
+  else:
+    tokenizer: CLIPTokenizer = CLIPTokenizer.from_pretrained(TOKENIZER_PATH)  # , model_max_length=max_token_length + 2)
+
+  text_encoder.to(DEVICE, dtype=weights_dtype)
+  text_encoder.eval()
+  unet.to(DEVICE, dtype=weights_dtype)
+  unet.eval()               # U-Netは呼び出さないので不要だけど
+
+  # トークンをひとつひとつ当たっていく
+  token_id_start = 0
+  token_id_end = max(tokenizer.all_special_ids)
+  print(f"interrogate tokens are: {token_id_start} to {token_id_end}")
+
+  def get_all_embeddings(text_encoder):
+    embs = []
+    with torch.no_grad():
+      for token_id in tqdm(range(token_id_start, token_id_end + 1, args.batch_size)):
+        batch = []
+        for tid in range(token_id, min(token_id_end + 1, token_id + args.batch_size)):
+          tokens = [tokenizer.bos_token_id, tid, tokenizer.eos_token_id]
+          # tokens = [tid]                                                    # こちらは結果がいまひとつ
+          batch.append(tokens)
+
+        # batch_embs = text_encoder(torch.tensor(batch).to(DEVICE))[0].to("cpu")  # bos/eosも含めたほうが差が出るようだ [:, 1]
+        # clip skip対応
+        batch = torch.tensor(batch).to(DEVICE)
+        if args.clip_skip is None:
+          encoder_hidden_states = text_encoder(batch)[0]
+        else:
+          enc_out = text_encoder(batch, output_hidden_states=True, return_dict=True)
+          encoder_hidden_states = enc_out['hidden_states'][-args.clip_skip]
+          encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
+        encoder_hidden_states = encoder_hidden_states.to("cpu")
+
+        embs.extend(encoder_hidden_states)
+    return torch.stack(embs)
+
+  print("get original text encoder embeddings.")
+  orig_embs = get_all_embeddings(text_encoder)
+
+  network.apply_to(text_encoder, unet, True, len(network.unet_loras) > 0)
+  info = network.load_state_dict(weights_sd, strict=False)
+  print(f"Loading LoRA weights: {info}")
+
+  network.to(DEVICE, dtype=weights_dtype)
+  network.eval()
+
+  del unet
+
+  print("You can ignore warning messages start with '_IncompatibleKeys' (LoRA model does not have alpha because trained by older script) / '_IncompatibleKeys'の警告は無視して構いません（以前のスクリプトで学習されたLoRAモデルのためalphaの定義がありません）")
+  print("get text encoder embeddings with lora.")
+  lora_embs = get_all_embeddings(text_encoder)
+
+  # 比べる：とりあえず単純に差分の絶対値で
+  print("comparing...")
+  diffs = {}
+  for i, (orig_emb, lora_emb) in enumerate(zip(orig_embs, tqdm(lora_embs))):
+    diff = torch.mean(torch.abs(orig_emb - lora_emb))
+    # diff = torch.mean(torch.cosine_similarity(orig_emb, lora_emb, dim=1))       # うまく検出できない
+    diff = float(diff.detach().to('cpu').numpy())
+    diffs[token_id_start + i] = diff
+
+  diffs_sorted = sorted(diffs.items(), key=lambda x: -x[1])
+
+  # 結果を表示する
+  print("top 100:")
+  for i, (token, diff) in enumerate(diffs_sorted[:100]):
+    # if diff < 1e-6:
+    #   break
+    string = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens([token]))
+    print(f"[{i:3d}]: {token:5d} {string:<20s}: {diff:.5f}")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument("--v2", action='store_true',
+                      help='load Stable Diffusion v2.x model / Stable Diffusion 2.xのモデルを読み込む')
+  parser.add_argument("--sd_model", type=str, default=None,
+                      help="Stable Diffusion model to load: ckpt or safetensors file / 読み込むSDのモデル、ckptまたはsafetensors")
+  parser.add_argument("--model", type=str, default=None,
+                      help="LoRA model to interrogate: ckpt or safetensors file / 調査するLoRAモデル、ckptまたはsafetensors")
+  parser.add_argument("--batch_size", type=int, default=16,
+                      help="batch size for processing with Text Encoder / Text Encoderで処理するときのバッチサイズ")
+  parser.add_argument("--clip_skip", type=int, default=None,
+                      help="use output of nth layer from back of text encoder (n>=1) / text encoderの後ろからn番目の層の出力を用いる（nは1以上）")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+  interrogate(args)
diff --git a/networks/merge_lora.py b/networks/merge_lora.py
new file mode 100644
index 0000000000000000000000000000000000000000..2fa8861bca3233a89640b9f02e75c8646a84171a
--- /dev/null
+++ b/networks/merge_lora.py
@@ -0,0 +1,243 @@
+import math
+import argparse
+import os
+import torch
+from safetensors.torch import load_file, save_file
+import library.model_util as model_util
+import lora
+
+
+def load_state_dict(file_name, dtype):
+    if os.path.splitext(file_name)[1] == ".safetensors":
+        sd = load_file(file_name)
+    else:
+        sd = torch.load(file_name, map_location="cpu")
+    for key in list(sd.keys()):
+        if type(sd[key]) == torch.Tensor:
+            sd[key] = sd[key].to(dtype)
+    return sd
+
+
+def save_to_file(file_name, model, state_dict, dtype):
+    if dtype is not None:
+        for key in list(state_dict.keys()):
+            if type(state_dict[key]) == torch.Tensor:
+                state_dict[key] = state_dict[key].to(dtype)
+
+    if os.path.splitext(file_name)[1] == ".safetensors":
+        save_file(model, file_name)
+    else:
+        torch.save(model, file_name)
+
+
+def merge_to_sd_model(text_encoder, unet, models, ratios, merge_dtype):
+    text_encoder.to(merge_dtype)
+    unet.to(merge_dtype)
+
+    # create module map
+    name_to_module = {}
+    for i, root_module in enumerate([text_encoder, unet]):
+        if i == 0:
+            prefix = lora.LoRANetwork.LORA_PREFIX_TEXT_ENCODER
+            target_replace_modules = lora.LoRANetwork.TEXT_ENCODER_TARGET_REPLACE_MODULE
+        else:
+            prefix = lora.LoRANetwork.LORA_PREFIX_UNET
+            target_replace_modules = (
+                lora.LoRANetwork.UNET_TARGET_REPLACE_MODULE + lora.LoRANetwork.UNET_TARGET_REPLACE_MODULE_CONV2D_3X3
+            )
+
+        for name, module in root_module.named_modules():
+            if module.__class__.__name__ in target_replace_modules:
+                for child_name, child_module in module.named_modules():
+                    if child_module.__class__.__name__ == "Linear" or child_module.__class__.__name__ == "Conv2d":
+                        lora_name = prefix + "." + name + "." + child_name
+                        lora_name = lora_name.replace(".", "_")
+                        name_to_module[lora_name] = child_module
+
+    for model, ratio in zip(models, ratios):
+        print(f"loading: {model}")
+        lora_sd = load_state_dict(model, merge_dtype)
+
+        print(f"merging...")
+        for key in lora_sd.keys():
+            if "lora_down" in key:
+                up_key = key.replace("lora_down", "lora_up")
+                alpha_key = key[: key.index("lora_down")] + "alpha"
+
+                # find original module for this lora
+                module_name = ".".join(key.split(".")[:-2])  # remove trailing ".lora_down.weight"
+                if module_name not in name_to_module:
+                    print(f"no module found for LoRA weight: {key}")
+                    continue
+                module = name_to_module[module_name]
+                # print(f"apply {key} to {module}")
+
+                down_weight = lora_sd[key]
+                up_weight = lora_sd[up_key]
+
+                dim = down_weight.size()[0]
+                alpha = lora_sd.get(alpha_key, dim)
+                scale = alpha / dim
+
+                # W <- W + U * D
+                weight = module.weight
+                # print(module_name, down_weight.size(), up_weight.size())
+                if len(weight.size()) == 2:
+                    # linear
+                    weight = weight + ratio * (up_weight @ down_weight) * scale
+                elif down_weight.size()[2:4] == (1, 1):
+                    # conv2d 1x1
+                    weight = (
+                        weight
+                        + ratio
+                        * (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3)
+                        * scale
+                    )
+                else:
+                    # conv2d 3x3
+                    conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3)
+                    # print(conved.size(), weight.size(), module.stride, module.padding)
+                    weight = weight + ratio * conved * scale
+
+                module.weight = torch.nn.Parameter(weight)
+
+
+def merge_lora_models(models, ratios, merge_dtype):
+    base_alphas = {}  # alpha for merged model
+    base_dims = {}
+
+    merged_sd = {}
+    for model, ratio in zip(models, ratios):
+        print(f"loading: {model}")
+        lora_sd = load_state_dict(model, merge_dtype)
+
+        # get alpha and dim
+        alphas = {}  # alpha for current model
+        dims = {}  # dims for current model
+        for key in lora_sd.keys():
+            if "alpha" in key:
+                lora_module_name = key[: key.rfind(".alpha")]
+                alpha = float(lora_sd[key].detach().numpy())
+                alphas[lora_module_name] = alpha
+                if lora_module_name not in base_alphas:
+                    base_alphas[lora_module_name] = alpha
+            elif "lora_down" in key:
+                lora_module_name = key[: key.rfind(".lora_down")]
+                dim = lora_sd[key].size()[0]
+                dims[lora_module_name] = dim
+                if lora_module_name not in base_dims:
+                    base_dims[lora_module_name] = dim
+
+        for lora_module_name in dims.keys():
+            if lora_module_name not in alphas:
+                alpha = dims[lora_module_name]
+                alphas[lora_module_name] = alpha
+                if lora_module_name not in base_alphas:
+                    base_alphas[lora_module_name] = alpha
+
+        print(f"dim: {list(set(dims.values()))}, alpha: {list(set(alphas.values()))}")
+
+        # merge
+        print(f"merging...")
+        for key in lora_sd.keys():
+            if "alpha" in key:
+                continue
+
+            lora_module_name = key[: key.rfind(".lora_")]
+
+            base_alpha = base_alphas[lora_module_name]
+            alpha = alphas[lora_module_name]
+
+            scale = math.sqrt(alpha / base_alpha) * ratio
+
+            if key in merged_sd:
+                assert (
+                    merged_sd[key].size() == lora_sd[key].size()
+                ), f"weights shape mismatch merging v1 and v2, different dims? / 重みのサイズが合いません。v1とv2、または次元数の異なるモデルはマージできません"
+                merged_sd[key] = merged_sd[key] + lora_sd[key] * scale
+            else:
+                merged_sd[key] = lora_sd[key] * scale
+
+    # set alpha to sd
+    for lora_module_name, alpha in base_alphas.items():
+        key = lora_module_name + ".alpha"
+        merged_sd[key] = torch.tensor(alpha)
+
+    print("merged model")
+    print(f"dim: {list(set(base_dims.values()))}, alpha: {list(set(base_alphas.values()))}")
+
+    return merged_sd
+
+
+def merge(args):
+    assert len(args.models) == len(args.ratios), f"number of models must be equal to number of ratios / モデルの数と重みの数は合わせてください"
+
+    def str_to_dtype(p):
+        if p == "float":
+            return torch.float
+        if p == "fp16":
+            return torch.float16
+        if p == "bf16":
+            return torch.bfloat16
+        return None
+
+    merge_dtype = str_to_dtype(args.precision)
+    save_dtype = str_to_dtype(args.save_precision)
+    if save_dtype is None:
+        save_dtype = merge_dtype
+
+    if args.sd_model is not None:
+        print(f"loading SD model: {args.sd_model}")
+
+        text_encoder, vae, unet = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.sd_model)
+
+        merge_to_sd_model(text_encoder, unet, args.models, args.ratios, merge_dtype)
+
+        print(f"saving SD model to: {args.save_to}")
+        model_util.save_stable_diffusion_checkpoint(args.v2, args.save_to, text_encoder, unet, args.sd_model, 0, 0, save_dtype, vae)
+    else:
+        state_dict = merge_lora_models(args.models, args.ratios, merge_dtype)
+
+        print(f"saving model to: {args.save_to}")
+        save_to_file(args.save_to, state_dict, state_dict, save_dtype)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--v2", action="store_true", help="load Stable Diffusion v2.x model / Stable Diffusion 2.xのモデルを読み込む")
+    parser.add_argument(
+        "--save_precision",
+        type=str,
+        default=None,
+        choices=[None, "float", "fp16", "bf16"],
+        help="precision in saving, same to merging if omitted / 保存時に精度を変更して保存する、省略時はマージ時の精度と同じ",
+    )
+    parser.add_argument(
+        "--precision",
+        type=str,
+        default="float",
+        choices=["float", "fp16", "bf16"],
+        help="precision in merging (float is recommended) / マージの計算時の精度（floatを推奨）",
+    )
+    parser.add_argument(
+        "--sd_model",
+        type=str,
+        default=None,
+        help="Stable Diffusion model to load: ckpt or safetensors file, merge LoRA models if omitted / 読み込むモデル、ckptまたはsafetensors。省略時はLoRAモデル同士をマージする",
+    )
+    parser.add_argument(
+        "--save_to", type=str, default=None, help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors"
+    )
+    parser.add_argument(
+        "--models", type=str, nargs="*", help="LoRA models to merge: ckpt or safetensors file / マージするLoRAモデル、ckptまたはsafetensors"
+    )
+    parser.add_argument("--ratios", type=float, nargs="*", help="ratios for each model / それぞれのLoRAモデルの比率")
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    merge(args)
diff --git a/networks/merge_lora_old.py b/networks/merge_lora_old.py
new file mode 100644
index 0000000000000000000000000000000000000000..ffd6b2b4095ec68a419e925908fe82ff69c5afad
--- /dev/null
+++ b/networks/merge_lora_old.py
@@ -0,0 +1,185 @@
+
+
+import argparse
+import os
+import torch
+from safetensors.torch import load_file, save_file
+import library.model_util as model_util
+import lora
+
+
+def load_state_dict(file_name, dtype):
+  if os.path.splitext(file_name)[1] == '.safetensors':
+    sd = load_file(file_name)
+  else:
+    sd = torch.load(file_name, map_location='cpu')
+  for key in list(sd.keys()):
+    if type(sd[key]) == torch.Tensor:
+      sd[key] = sd[key].to(dtype)
+  return sd
+
+
+def save_to_file(file_name, model, state_dict, dtype):
+  if dtype is not None:
+    for key in list(state_dict.keys()):
+      if type(state_dict[key]) == torch.Tensor:
+        state_dict[key] = state_dict[key].to(dtype)
+
+  if os.path.splitext(file_name)[1] == '.safetensors':
+    save_file(model, file_name)
+  else:
+    torch.save(model, file_name)
+
+
+def merge_to_sd_model(text_encoder, unet, models, ratios, merge_dtype):
+  text_encoder.to(merge_dtype)
+  unet.to(merge_dtype)
+
+  # create module map
+  name_to_module = {}
+  for i, root_module in enumerate([text_encoder, unet]):
+    if i == 0:
+      prefix = lora.LoRANetwork.LORA_PREFIX_TEXT_ENCODER
+      target_replace_modules = lora.LoRANetwork.TEXT_ENCODER_TARGET_REPLACE_MODULE
+    else:
+      prefix = lora.LoRANetwork.LORA_PREFIX_UNET
+      target_replace_modules = lora.LoRANetwork.UNET_TARGET_REPLACE_MODULE
+
+    for name, module in root_module.named_modules():
+      if module.__class__.__name__ in target_replace_modules:
+        for child_name, child_module in module.named_modules():
+          if child_module.__class__.__name__ == "Linear" or (child_module.__class__.__name__ == "Conv2d" and child_module.kernel_size == (1, 1)):
+            lora_name = prefix + '.' + name + '.' + child_name
+            lora_name = lora_name.replace('.', '_')
+            name_to_module[lora_name] = child_module
+
+  for model, ratio in zip(models, ratios):
+    print(f"loading: {model}")
+    lora_sd = load_state_dict(model, merge_dtype)
+
+    print(f"merging...")
+    for key in lora_sd.keys():
+      if "lora_down" in key:
+        up_key = key.replace("lora_down", "lora_up")
+        alpha_key = key[:key.index("lora_down")] + 'alpha'
+
+        # find original module for this lora
+        module_name = '.'.join(key.split('.')[:-2])               # remove trailing ".lora_down.weight"
+        if module_name not in name_to_module:
+          print(f"no module found for LoRA weight: {key}")
+          continue
+        module = name_to_module[module_name]
+        # print(f"apply {key} to {module}")
+
+        down_weight = lora_sd[key]
+        up_weight = lora_sd[up_key]
+
+        dim = down_weight.size()[0]
+        alpha = lora_sd.get(alpha_key, dim)
+        scale = alpha / dim
+
+        # W <- W + U * D
+        weight = module.weight
+        if len(weight.size()) == 2:
+          # linear
+          weight = weight + ratio * (up_weight @ down_weight) * scale
+        else:
+          # conv2d
+          weight = weight + ratio * (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3) * scale
+
+        module.weight = torch.nn.Parameter(weight)
+
+
+def merge_lora_models(models, ratios, merge_dtype):
+  merged_sd = {}
+
+  alpha = None
+  dim = None
+  for model, ratio in zip(models, ratios):
+    print(f"loading: {model}")
+    lora_sd = load_state_dict(model, merge_dtype)
+
+    print(f"merging...")
+    for key in lora_sd.keys():
+      if 'alpha' in key:
+        if key in merged_sd:
+          assert merged_sd[key] == lora_sd[key], f"alpha mismatch / alphaが異なる場合、現時点ではマージできません"
+        else:
+          alpha = lora_sd[key].detach().numpy()
+          merged_sd[key] = lora_sd[key]
+      else:
+        if key in merged_sd:
+          assert merged_sd[key].size() == lora_sd[key].size(
+          ), f"weights shape mismatch merging v1 and v2, different dims? / 重みのサイズが合いません。v1とv2、または次元数の異なるモデルはマージできません"
+          merged_sd[key] = merged_sd[key] + lora_sd[key] * ratio
+        else:
+          if "lora_down" in key:
+            dim = lora_sd[key].size()[0]
+          merged_sd[key] = lora_sd[key] * ratio
+
+  print(f"dim (rank): {dim}, alpha: {alpha}")
+  if alpha is None:
+    alpha = dim
+
+  return merged_sd, dim, alpha
+
+
+def merge(args):
+  assert len(args.models) == len(args.ratios), f"number of models must be equal to number of ratios / モデルの数と重みの数は合わせてください"
+
+  def str_to_dtype(p):
+    if p == 'float':
+      return torch.float
+    if p == 'fp16':
+      return torch.float16
+    if p == 'bf16':
+      return torch.bfloat16
+    return None
+
+  merge_dtype = str_to_dtype(args.precision)
+  save_dtype = str_to_dtype(args.save_precision)
+  if save_dtype is None:
+    save_dtype = merge_dtype
+
+  if args.sd_model is not None:
+    print(f"loading SD model: {args.sd_model}")
+
+    text_encoder, vae, unet = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.sd_model)
+
+    merge_to_sd_model(text_encoder, unet, args.models, args.ratios, merge_dtype)
+
+    print(f"\nsaving SD model to: {args.save_to}")
+    model_util.save_stable_diffusion_checkpoint(args.v2, args.save_to, text_encoder, unet,
+                                                args.sd_model, 0, 0, save_dtype, vae)
+  else:
+    state_dict, _, _ = merge_lora_models(args.models, args.ratios, merge_dtype)
+
+    print(f"\nsaving model to: {args.save_to}")
+    save_to_file(args.save_to, state_dict, state_dict, save_dtype)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--v2", action='store_true',
+                      help='load Stable Diffusion v2.x model / Stable Diffusion 2.xのモデルを読み込む')
+  parser.add_argument("--save_precision", type=str, default=None,
+                      choices=[None, "float", "fp16", "bf16"], help="precision in saving, same to merging if omitted / 保存時に精度を変更して保存する、省略時はマージ時の精度と同じ")
+  parser.add_argument("--precision", type=str, default="float",
+                      choices=["float", "fp16", "bf16"], help="precision in merging (float is recommended) / マージの計算時の精度（floatを推奨）")
+  parser.add_argument("--sd_model", type=str, default=None,
+                      help="Stable Diffusion model to load: ckpt or safetensors file, merge LoRA models if omitted / 読み込むモデル、ckptまたはsafetensors。省略時はLoRAモデル同士をマージする")
+  parser.add_argument("--save_to", type=str, default=None,
+                      help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors")
+  parser.add_argument("--models", type=str, nargs='*',
+                      help="LoRA models to merge: ckpt or safetensors file / マージするLoRAモデル、ckptまたはsafetensors")
+  parser.add_argument("--ratios", type=float, nargs='*',
+                      help="ratios for each model / それぞれのLoRAモデルの比率")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+  merge(args)
diff --git a/networks/resize_lora.py b/networks/resize_lora.py
new file mode 100644
index 0000000000000000000000000000000000000000..7b7406347d77d05b8bbf3797577a28d07d60b975
--- /dev/null
+++ b/networks/resize_lora.py
@@ -0,0 +1,359 @@
+# Convert LoRA to different rank approximation (should only be used to go to lower rank)
+# This code is based off the extract_lora_from_models.py file which is based on https://github.com/cloneofsimo/lora/blob/develop/lora_diffusion/cli_svd.py
+# Thanks to cloneofsimo
+
+import argparse
+import torch
+from safetensors.torch import load_file, save_file, safe_open
+from tqdm import tqdm
+from library import train_util, model_util
+import numpy as np
+
+MIN_SV = 1e-6
+
+# Model save and load functions
+
+def load_state_dict(file_name, dtype):
+  if model_util.is_safetensors(file_name):
+    sd = load_file(file_name)
+    with safe_open(file_name, framework="pt") as f:
+      metadata = f.metadata()
+  else:
+    sd = torch.load(file_name, map_location='cpu')
+    metadata = None
+
+  for key in list(sd.keys()):
+    if type(sd[key]) == torch.Tensor:
+      sd[key] = sd[key].to(dtype)
+
+  return sd, metadata
+
+
+def save_to_file(file_name, model, state_dict, dtype, metadata):
+  if dtype is not None:
+    for key in list(state_dict.keys()):
+      if type(state_dict[key]) == torch.Tensor:
+        state_dict[key] = state_dict[key].to(dtype)
+
+  if model_util.is_safetensors(file_name):
+    save_file(model, file_name, metadata)
+  else:
+    torch.save(model, file_name)
+
+
+# Indexing functions
+
+def index_sv_cumulative(S, target):
+  original_sum = float(torch.sum(S))
+  cumulative_sums = torch.cumsum(S, dim=0)/original_sum
+  index = int(torch.searchsorted(cumulative_sums, target)) + 1
+  index = max(1, min(index, len(S)-1))
+
+  return index
+
+
+def index_sv_fro(S, target):
+  S_squared = S.pow(2)
+  s_fro_sq = float(torch.sum(S_squared))
+  sum_S_squared = torch.cumsum(S_squared, dim=0)/s_fro_sq
+  index = int(torch.searchsorted(sum_S_squared, target**2)) + 1
+  index = max(1, min(index, len(S)-1))
+
+  return index
+
+
+def index_sv_ratio(S, target):
+  max_sv = S[0]
+  min_sv = max_sv/target
+  index = int(torch.sum(S > min_sv).item())
+  index = max(1, min(index, len(S)-1))
+
+  return index
+
+
+# Modified from Kohaku-blueleaf's extract/merge functions
+def extract_conv(weight, lora_rank, dynamic_method, dynamic_param, device, scale=1):
+    out_size, in_size, kernel_size, _ = weight.size()
+    U, S, Vh = torch.linalg.svd(weight.reshape(out_size, -1).to(device))
+    
+    param_dict = rank_resize(S, lora_rank, dynamic_method, dynamic_param, scale)
+    lora_rank = param_dict["new_rank"]
+
+    U = U[:, :lora_rank]
+    S = S[:lora_rank]
+    U = U @ torch.diag(S)
+    Vh = Vh[:lora_rank, :]
+
+    param_dict["lora_down"] = Vh.reshape(lora_rank, in_size, kernel_size, kernel_size).cpu()
+    param_dict["lora_up"] = U.reshape(out_size, lora_rank, 1, 1).cpu()
+    del U, S, Vh, weight
+    return param_dict
+
+
+def extract_linear(weight, lora_rank, dynamic_method, dynamic_param, device, scale=1):
+    out_size, in_size = weight.size()
+    
+    U, S, Vh = torch.linalg.svd(weight.to(device))
+    
+    param_dict = rank_resize(S, lora_rank, dynamic_method, dynamic_param, scale)
+    lora_rank = param_dict["new_rank"]
+    
+    U = U[:, :lora_rank]
+    S = S[:lora_rank]
+    U = U @ torch.diag(S)
+    Vh = Vh[:lora_rank, :]
+    
+    param_dict["lora_down"] = Vh.reshape(lora_rank, in_size).cpu()
+    param_dict["lora_up"] = U.reshape(out_size, lora_rank).cpu()
+    del U, S, Vh, weight
+    return param_dict
+
+
+def merge_conv(lora_down, lora_up, device):
+    in_rank, in_size, kernel_size, k_ = lora_down.shape
+    out_size, out_rank, _, _ = lora_up.shape
+    assert in_rank == out_rank and kernel_size == k_, f"rank {in_rank} {out_rank} or kernel {kernel_size} {k_} mismatch"
+    
+    lora_down = lora_down.to(device)
+    lora_up = lora_up.to(device)
+
+    merged = lora_up.reshape(out_size, -1) @ lora_down.reshape(in_rank, -1)
+    weight = merged.reshape(out_size, in_size, kernel_size, kernel_size)
+    del lora_up, lora_down
+    return weight
+
+
+def merge_linear(lora_down, lora_up, device):
+    in_rank, in_size = lora_down.shape
+    out_size, out_rank = lora_up.shape
+    assert in_rank == out_rank, f"rank {in_rank} {out_rank} mismatch"
+    
+    lora_down = lora_down.to(device)
+    lora_up = lora_up.to(device)
+    
+    weight = lora_up @ lora_down
+    del lora_up, lora_down
+    return weight
+  
+
+# Calculate new rank
+
+def rank_resize(S, rank, dynamic_method, dynamic_param, scale=1):
+    param_dict = {}
+
+    if dynamic_method=="sv_ratio":
+        # Calculate new dim and alpha based off ratio
+        new_rank = index_sv_ratio(S, dynamic_param) + 1
+        new_alpha = float(scale*new_rank)
+
+    elif dynamic_method=="sv_cumulative":
+        # Calculate new dim and alpha based off cumulative sum
+        new_rank = index_sv_cumulative(S, dynamic_param) + 1
+        new_alpha = float(scale*new_rank)
+
+    elif dynamic_method=="sv_fro":
+        # Calculate new dim and alpha based off sqrt sum of squares
+        new_rank = index_sv_fro(S, dynamic_param) + 1
+        new_alpha = float(scale*new_rank)
+    else:
+        new_rank = rank
+        new_alpha = float(scale*new_rank)
+
+    
+    if S[0] <= MIN_SV: # Zero matrix, set dim to 1
+        new_rank = 1
+        new_alpha = float(scale*new_rank)
+    elif new_rank > rank: # cap max rank at rank
+        new_rank = rank
+        new_alpha = float(scale*new_rank)
+
+
+    # Calculate resize info
+    s_sum = torch.sum(torch.abs(S))
+    s_rank = torch.sum(torch.abs(S[:new_rank]))
+    
+    S_squared = S.pow(2)
+    s_fro = torch.sqrt(torch.sum(S_squared))
+    s_red_fro = torch.sqrt(torch.sum(S_squared[:new_rank]))
+    fro_percent = float(s_red_fro/s_fro)
+
+    param_dict["new_rank"] = new_rank
+    param_dict["new_alpha"] = new_alpha
+    param_dict["sum_retained"] = (s_rank)/s_sum
+    param_dict["fro_retained"] = fro_percent
+    param_dict["max_ratio"] = S[0]/S[new_rank - 1]
+
+    return param_dict
+
+
+def resize_lora_model(lora_sd, new_rank, save_dtype, device, dynamic_method, dynamic_param, verbose):
+  network_alpha = None
+  network_dim = None
+  verbose_str = "\n"
+  fro_list = []
+
+  # Extract loaded lora dim and alpha
+  for key, value in lora_sd.items():
+    if network_alpha is None and 'alpha' in key:
+      network_alpha = value
+    if network_dim is None and 'lora_down' in key and len(value.size()) == 2:
+      network_dim = value.size()[0]
+    if network_alpha is not None and network_dim is not None:
+      break
+    if network_alpha is None:
+      network_alpha = network_dim
+
+  scale = network_alpha/network_dim
+
+  if dynamic_method:
+    print(f"Dynamically determining new alphas and dims based off {dynamic_method}: {dynamic_param}, max rank is {new_rank}")
+
+  lora_down_weight = None
+  lora_up_weight = None
+
+  o_lora_sd = lora_sd.copy()
+  block_down_name = None
+  block_up_name = None
+
+  with torch.no_grad():
+    for key, value in tqdm(lora_sd.items()):
+      weight_name = None
+      if 'lora_down' in key:
+        block_down_name = key.split(".")[0]
+        weight_name = key.split(".")[-1]
+        lora_down_weight = value
+      else:
+        continue
+
+      # find corresponding lora_up and alpha
+      block_up_name = block_down_name
+      lora_up_weight = lora_sd.get(block_up_name + '.lora_up.' + weight_name, None)
+      lora_alpha = lora_sd.get(block_down_name + '.alpha', None)
+
+      weights_loaded = (lora_down_weight is not None and lora_up_weight is not None)
+
+      if weights_loaded:
+
+        conv2d = (len(lora_down_weight.size()) == 4)
+        if lora_alpha is None:
+          scale = 1.0
+        else:
+          scale = lora_alpha/lora_down_weight.size()[0]
+
+        if conv2d:
+          full_weight_matrix = merge_conv(lora_down_weight, lora_up_weight, device)
+          param_dict = extract_conv(full_weight_matrix, new_rank, dynamic_method, dynamic_param, device, scale)
+        else:
+          full_weight_matrix = merge_linear(lora_down_weight, lora_up_weight, device)
+          param_dict = extract_linear(full_weight_matrix, new_rank, dynamic_method, dynamic_param, device, scale)
+
+        if verbose:
+          max_ratio = param_dict['max_ratio']
+          sum_retained = param_dict['sum_retained']
+          fro_retained = param_dict['fro_retained']
+          if not np.isnan(fro_retained):
+            fro_list.append(float(fro_retained))
+
+          verbose_str+=f"{block_down_name:75} | "
+          verbose_str+=f"sum(S) retained: {sum_retained:.1%}, fro retained: {fro_retained:.1%}, max(S) ratio: {max_ratio:0.1f}"
+
+        if verbose and dynamic_method:
+          verbose_str+=f", dynamic | dim: {param_dict['new_rank']}, alpha: {param_dict['new_alpha']}\n"
+        else:
+          verbose_str+=f"\n"
+
+        new_alpha = param_dict['new_alpha']
+        o_lora_sd[block_down_name + "." + "lora_down.weight"] = param_dict["lora_down"].to(save_dtype).contiguous()
+        o_lora_sd[block_up_name + "." + "lora_up.weight"] = param_dict["lora_up"].to(save_dtype).contiguous()
+        o_lora_sd[block_up_name + "." "alpha"] = torch.tensor(param_dict['new_alpha']).to(save_dtype)
+
+        block_down_name = None
+        block_up_name = None
+        lora_down_weight = None
+        lora_up_weight = None
+        weights_loaded = False
+        del param_dict
+
+  if verbose:
+    print(verbose_str)
+
+    print(f"Average Frobenius norm retention: {np.mean(fro_list):.2%} | std: {np.std(fro_list):0.3f}")
+  print("resizing complete")
+  return o_lora_sd, network_dim, new_alpha
+
+
+def resize(args):
+
+  def str_to_dtype(p):
+    if p == 'float':
+      return torch.float
+    if p == 'fp16':
+      return torch.float16
+    if p == 'bf16':
+      return torch.bfloat16
+    return None
+
+  if args.dynamic_method and not args.dynamic_param:
+    raise Exception("If using dynamic_method, then dynamic_param is required")
+
+  merge_dtype = str_to_dtype('float')  # matmul method above only seems to work in float32
+  save_dtype = str_to_dtype(args.save_precision)
+  if save_dtype is None:
+    save_dtype = merge_dtype
+
+  print("loading Model...")
+  lora_sd, metadata = load_state_dict(args.model, merge_dtype)
+
+  print("Resizing Lora...")
+  state_dict, old_dim, new_alpha = resize_lora_model(lora_sd, args.new_rank, save_dtype, args.device, args.dynamic_method, args.dynamic_param, args.verbose)
+
+  # update metadata
+  if metadata is None:
+    metadata = {}
+
+  comment = metadata.get("ss_training_comment", "")
+
+  if not args.dynamic_method:
+    metadata["ss_training_comment"] = f"dimension is resized from {old_dim} to {args.new_rank}; {comment}"
+    metadata["ss_network_dim"] = str(args.new_rank)
+    metadata["ss_network_alpha"] = str(new_alpha)
+  else:
+    metadata["ss_training_comment"] = f"Dynamic resize with {args.dynamic_method}: {args.dynamic_param} from {old_dim}; {comment}"
+    metadata["ss_network_dim"] = 'Dynamic'
+    metadata["ss_network_alpha"] = 'Dynamic'
+
+  model_hash, legacy_hash = train_util.precalculate_safetensors_hashes(state_dict, metadata)
+  metadata["sshs_model_hash"] = model_hash
+  metadata["sshs_legacy_hash"] = legacy_hash
+
+  print(f"saving model to: {args.save_to}")
+  save_to_file(args.save_to, state_dict, state_dict, save_dtype, metadata)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument("--save_precision", type=str, default=None,
+                      choices=[None, "float", "fp16", "bf16"], help="precision in saving, float if omitted / 保存時の精度、未指定時はfloat")
+  parser.add_argument("--new_rank", type=int, default=4,
+                      help="Specify rank of output LoRA / 出力するLoRAのrank (dim)")
+  parser.add_argument("--save_to", type=str, default=None,
+                      help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors")
+  parser.add_argument("--model", type=str, default=None,
+                      help="LoRA model to resize at to new rank: ckpt or safetensors file / 読み込むLoRAモデル、ckptまたはsafetensors")
+  parser.add_argument("--device", type=str, default=None, help="device to use, cuda for GPU / 計算を行うデバイス、cuda でGPUを使う")
+  parser.add_argument("--verbose", action="store_true", 
+                      help="Display verbose resizing information / rank変更時の詳細情報を出力する")
+  parser.add_argument("--dynamic_method", type=str, default=None, choices=[None, "sv_ratio", "sv_fro", "sv_cumulative"],
+                      help="Specify dynamic resizing method, --new_rank is used as a hard limit for max rank")
+  parser.add_argument("--dynamic_param", type=float, default=None,
+                      help="Specify target for dynamic reduction")
+       
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+  resize(args)
diff --git a/networks/sdxl_merge_lora.py b/networks/sdxl_merge_lora.py
new file mode 100644
index 0000000000000000000000000000000000000000..d75da7d7677403ae1cbc45c071724b79c4ff26cc
--- /dev/null
+++ b/networks/sdxl_merge_lora.py
@@ -0,0 +1,257 @@
+import math
+import argparse
+import os
+import torch
+from safetensors.torch import load_file, save_file
+from tqdm import tqdm
+from library import sdxl_model_util
+import library.model_util as model_util
+import lora
+
+
+def load_state_dict(file_name, dtype):
+    if os.path.splitext(file_name)[1] == ".safetensors":
+        sd = load_file(file_name)
+    else:
+        sd = torch.load(file_name, map_location="cpu")
+    for key in list(sd.keys()):
+        if type(sd[key]) == torch.Tensor:
+            sd[key] = sd[key].to(dtype)
+    return sd
+
+
+def save_to_file(file_name, model, state_dict, dtype):
+    if dtype is not None:
+        for key in list(state_dict.keys()):
+            if type(state_dict[key]) == torch.Tensor:
+                state_dict[key] = state_dict[key].to(dtype)
+
+    if os.path.splitext(file_name)[1] == ".safetensors":
+        save_file(model, file_name)
+    else:
+        torch.save(model, file_name)
+
+
+def merge_to_sd_model(text_encoder1, text_encoder2, unet, models, ratios, merge_dtype):
+    text_encoder1.to(merge_dtype)
+    text_encoder1.to(merge_dtype)
+    unet.to(merge_dtype)
+
+    # create module map
+    name_to_module = {}
+    for i, root_module in enumerate([text_encoder1, text_encoder2, unet]):
+        if i <= 1:
+            if i == 0:
+                prefix = lora.LoRANetwork.LORA_PREFIX_TEXT_ENCODER1
+            else:
+                prefix = lora.LoRANetwork.LORA_PREFIX_TEXT_ENCODER2
+            target_replace_modules = lora.LoRANetwork.TEXT_ENCODER_TARGET_REPLACE_MODULE
+        else:
+            prefix = lora.LoRANetwork.LORA_PREFIX_UNET
+            target_replace_modules = (
+                lora.LoRANetwork.UNET_TARGET_REPLACE_MODULE + lora.LoRANetwork.UNET_TARGET_REPLACE_MODULE_CONV2D_3X3
+            )
+
+        for name, module in root_module.named_modules():
+            if module.__class__.__name__ in target_replace_modules:
+                for child_name, child_module in module.named_modules():
+                    if child_module.__class__.__name__ == "Linear" or child_module.__class__.__name__ == "Conv2d":
+                        lora_name = prefix + "." + name + "." + child_name
+                        lora_name = lora_name.replace(".", "_")
+                        name_to_module[lora_name] = child_module
+
+    for model, ratio in zip(models, ratios):
+        print(f"loading: {model}")
+        lora_sd = load_state_dict(model, merge_dtype)
+
+        print(f"merging...")
+        for key in tqdm(lora_sd.keys()):
+            if "lora_down" in key:
+                up_key = key.replace("lora_down", "lora_up")
+                alpha_key = key[: key.index("lora_down")] + "alpha"
+
+                # find original module for this lora
+                module_name = ".".join(key.split(".")[:-2])  # remove trailing ".lora_down.weight"
+                if module_name not in name_to_module:
+                    print(f"no module found for LoRA weight: {key}")
+                    continue
+                module = name_to_module[module_name]
+                # print(f"apply {key} to {module}")
+
+                down_weight = lora_sd[key]
+                up_weight = lora_sd[up_key]
+
+                dim = down_weight.size()[0]
+                alpha = lora_sd.get(alpha_key, dim)
+                scale = alpha / dim
+
+                # W <- W + U * D
+                weight = module.weight
+                # print(module_name, down_weight.size(), up_weight.size())
+                if len(weight.size()) == 2:
+                    # linear
+                    weight = weight + ratio * (up_weight @ down_weight) * scale
+                elif down_weight.size()[2:4] == (1, 1):
+                    # conv2d 1x1
+                    weight = (
+                        weight
+                        + ratio
+                        * (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3)
+                        * scale
+                    )
+                else:
+                    # conv2d 3x3
+                    conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3)
+                    # print(conved.size(), weight.size(), module.stride, module.padding)
+                    weight = weight + ratio * conved * scale
+
+                module.weight = torch.nn.Parameter(weight)
+
+
+def merge_lora_models(models, ratios, merge_dtype):
+    base_alphas = {}  # alpha for merged model
+    base_dims = {}
+
+    merged_sd = {}
+    for model, ratio in zip(models, ratios):
+        print(f"loading: {model}")
+        lora_sd = load_state_dict(model, merge_dtype)
+
+        # get alpha and dim
+        alphas = {}  # alpha for current model
+        dims = {}  # dims for current model
+        for key in lora_sd.keys():
+            if "alpha" in key:
+                lora_module_name = key[: key.rfind(".alpha")]
+                alpha = float(lora_sd[key].detach().numpy())
+                alphas[lora_module_name] = alpha
+                if lora_module_name not in base_alphas:
+                    base_alphas[lora_module_name] = alpha
+            elif "lora_down" in key:
+                lora_module_name = key[: key.rfind(".lora_down")]
+                dim = lora_sd[key].size()[0]
+                dims[lora_module_name] = dim
+                if lora_module_name not in base_dims:
+                    base_dims[lora_module_name] = dim
+
+        for lora_module_name in dims.keys():
+            if lora_module_name not in alphas:
+                alpha = dims[lora_module_name]
+                alphas[lora_module_name] = alpha
+                if lora_module_name not in base_alphas:
+                    base_alphas[lora_module_name] = alpha
+
+        print(f"dim: {list(set(dims.values()))}, alpha: {list(set(alphas.values()))}")
+
+        # merge
+        print(f"merging...")
+        for key in tqdm(lora_sd.keys()):
+            if "alpha" in key:
+                continue
+
+            lora_module_name = key[: key.rfind(".lora_")]
+
+            base_alpha = base_alphas[lora_module_name]
+            alpha = alphas[lora_module_name]
+
+            scale = math.sqrt(alpha / base_alpha) * ratio
+
+            if key in merged_sd:
+                assert (
+                    merged_sd[key].size() == lora_sd[key].size()
+                ), f"weights shape mismatch merging v1 and v2, different dims? / 重みのサイズが合いません。v1とv2、または次元数の異なるモデルはマージできません"
+                merged_sd[key] = merged_sd[key] + lora_sd[key] * scale
+            else:
+                merged_sd[key] = lora_sd[key] * scale
+
+    # set alpha to sd
+    for lora_module_name, alpha in base_alphas.items():
+        key = lora_module_name + ".alpha"
+        merged_sd[key] = torch.tensor(alpha)
+
+    print("merged model")
+    print(f"dim: {list(set(base_dims.values()))}, alpha: {list(set(base_alphas.values()))}")
+
+    return merged_sd
+
+
+def merge(args):
+    assert len(args.models) == len(args.ratios), f"number of models must be equal to number of ratios / モデルの数と重みの数は合わせてください"
+
+    def str_to_dtype(p):
+        if p == "float":
+            return torch.float
+        if p == "fp16":
+            return torch.float16
+        if p == "bf16":
+            return torch.bfloat16
+        return None
+
+    merge_dtype = str_to_dtype(args.precision)
+    save_dtype = str_to_dtype(args.save_precision)
+    if save_dtype is None:
+        save_dtype = merge_dtype
+
+    if args.sd_model is not None:
+        print(f"loading SD model: {args.sd_model}")
+
+        (
+            text_model1,
+            text_model2,
+            vae,
+            unet,
+            logit_scale,
+            ckpt_info,
+        ) = sdxl_model_util.load_models_from_sdxl_checkpoint(sdxl_model_util.MODEL_VERSION_SDXL_BASE_V0_9, args.sd_model, "cpu")
+
+        merge_to_sd_model(text_model2, text_model2, unet, args.models, args.ratios, merge_dtype)
+
+        print(f"saving SD model to: {args.save_to}")
+        sdxl_model_util.save_stable_diffusion_checkpoint(
+            args.save_to, text_model1, text_model2, unet, 0, 0, ckpt_info, vae, logit_scale, save_dtype
+        )
+    else:
+        state_dict = merge_lora_models(args.models, args.ratios, merge_dtype)
+
+        print(f"saving model to: {args.save_to}")
+        save_to_file(args.save_to, state_dict, state_dict, save_dtype)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--save_precision",
+        type=str,
+        default=None,
+        choices=[None, "float", "fp16", "bf16"],
+        help="precision in saving, same to merging if omitted / 保存時に精度を変更して保存する、省略時はマージ時の精度と同じ",
+    )
+    parser.add_argument(
+        "--precision",
+        type=str,
+        default="float",
+        choices=["float", "fp16", "bf16"],
+        help="precision in merging (float is recommended) / マージの計算時の精度（floatを推奨）",
+    )
+    parser.add_argument(
+        "--sd_model",
+        type=str,
+        default=None,
+        help="Stable Diffusion model to load: ckpt or safetensors file, merge LoRA models if omitted / 読み込むモデル、ckptまたはsafetensors。省略時はLoRAモデル同士をマージする",
+    )
+    parser.add_argument(
+        "--save_to", type=str, default=None, help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors"
+    )
+    parser.add_argument(
+        "--models", type=str, nargs="*", help="LoRA models to merge: ckpt or safetensors file / マージするLoRAモデル、ckptまたはsafetensors"
+    )
+    parser.add_argument("--ratios", type=float, nargs="*", help="ratios for each model / それぞれのLoRAモデルの比率")
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    merge(args)
diff --git a/networks/svd_merge_lora.py b/networks/svd_merge_lora.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d17efba5e80fe0b75fcdb43e0ef6a7bb2ca5d7e
--- /dev/null
+++ b/networks/svd_merge_lora.py
@@ -0,0 +1,192 @@
+
+import math
+import argparse
+import os
+import torch
+from safetensors.torch import load_file, save_file
+from tqdm import tqdm
+import library.model_util as model_util
+import lora
+
+
+CLAMP_QUANTILE = 0.99
+
+
+def load_state_dict(file_name, dtype):
+  if os.path.splitext(file_name)[1] == '.safetensors':
+    sd = load_file(file_name)
+  else:
+    sd = torch.load(file_name, map_location='cpu')
+  for key in list(sd.keys()):
+    if type(sd[key]) == torch.Tensor:
+      sd[key] = sd[key].to(dtype)
+  return sd
+
+
+def save_to_file(file_name, state_dict, dtype):
+  if dtype is not None:
+    for key in list(state_dict.keys()):
+      if type(state_dict[key]) == torch.Tensor:
+        state_dict[key] = state_dict[key].to(dtype)
+
+  if os.path.splitext(file_name)[1] == '.safetensors':
+    save_file(state_dict, file_name)
+  else:
+    torch.save(state_dict, file_name)
+
+
+def merge_lora_models(models, ratios, new_rank, new_conv_rank, device, merge_dtype):
+  print(f"new rank: {new_rank}, new conv rank: {new_conv_rank}")
+  merged_sd = {}
+  for model, ratio in zip(models, ratios):
+    print(f"loading: {model}")
+    lora_sd = load_state_dict(model, merge_dtype)
+
+    # merge
+    print(f"merging...")
+    for key in tqdm(list(lora_sd.keys())):
+      if 'lora_down' not in key:
+        continue
+
+      lora_module_name = key[:key.rfind(".lora_down")]
+
+      down_weight = lora_sd[key]
+      network_dim = down_weight.size()[0]
+
+      up_weight = lora_sd[lora_module_name + '.lora_up.weight']
+      alpha = lora_sd.get(lora_module_name + '.alpha', network_dim)
+
+      in_dim = down_weight.size()[1]
+      out_dim = up_weight.size()[0]
+      conv2d = len(down_weight.size()) == 4
+      kernel_size = None if not conv2d else down_weight.size()[2:4]
+      # print(lora_module_name, network_dim, alpha, in_dim, out_dim, kernel_size)
+
+      # make original weight if not exist
+      if lora_module_name not in merged_sd:
+        weight = torch.zeros((out_dim, in_dim, *kernel_size) if conv2d else (out_dim, in_dim), dtype=merge_dtype)
+        if device:
+          weight = weight.to(device)
+      else:
+        weight = merged_sd[lora_module_name]
+
+      # merge to weight
+      if device:
+        up_weight = up_weight.to(device)
+        down_weight = down_weight.to(device)
+
+      # W <- W + U * D
+      scale = (alpha / network_dim)
+
+      if device:                      # and isinstance(scale, torch.Tensor):
+        scale = scale.to(device)
+
+      if not conv2d:        # linear
+        weight = weight + ratio * (up_weight @ down_weight) * scale
+      elif kernel_size == (1, 1):
+        weight = weight + ratio * (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)
+                                   ).unsqueeze(2).unsqueeze(3) * scale
+      else:
+        conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3)
+        weight = weight + ratio * conved * scale
+
+      merged_sd[lora_module_name] = weight
+
+  # extract from merged weights
+  print("extract new lora...")
+  merged_lora_sd = {}
+  with torch.no_grad():
+    for lora_module_name, mat in tqdm(list(merged_sd.items())):
+      conv2d = (len(mat.size()) == 4)
+      kernel_size = None if not conv2d else mat.size()[2:4]
+      conv2d_3x3 = conv2d and kernel_size != (1, 1)
+      out_dim, in_dim = mat.size()[0:2]
+
+      if conv2d:
+        if conv2d_3x3:
+          mat = mat.flatten(start_dim=1)
+        else:
+          mat = mat.squeeze()
+
+      module_new_rank = new_conv_rank if conv2d_3x3 else new_rank
+      module_new_rank = min(module_new_rank, in_dim, out_dim)                           # LoRA rank cannot exceed the original dim
+
+      U, S, Vh = torch.linalg.svd(mat)
+
+      U = U[:, :module_new_rank]
+      S = S[:module_new_rank]
+      U = U @ torch.diag(S)
+
+      Vh = Vh[:module_new_rank, :]
+
+      dist = torch.cat([U.flatten(), Vh.flatten()])
+      hi_val = torch.quantile(dist, CLAMP_QUANTILE)
+      low_val = -hi_val
+
+      U = U.clamp(low_val, hi_val)
+      Vh = Vh.clamp(low_val, hi_val)
+
+      if conv2d:
+        U = U.reshape(out_dim, module_new_rank, 1, 1)
+        Vh = Vh.reshape(module_new_rank, in_dim, kernel_size[0], kernel_size[1])
+
+      up_weight = U
+      down_weight = Vh
+
+      merged_lora_sd[lora_module_name + '.lora_up.weight'] = up_weight.to("cpu").contiguous()
+      merged_lora_sd[lora_module_name + '.lora_down.weight'] = down_weight.to("cpu").contiguous()
+      merged_lora_sd[lora_module_name + '.alpha'] = torch.tensor(module_new_rank)
+
+  return merged_lora_sd
+
+
+def merge(args):
+  assert len(args.models) == len(args.ratios), f"number of models must be equal to number of ratios / モデルの数と重みの数は合わせてください"
+
+  def str_to_dtype(p):
+    if p == 'float':
+      return torch.float
+    if p == 'fp16':
+      return torch.float16
+    if p == 'bf16':
+      return torch.bfloat16
+    return None
+
+  merge_dtype = str_to_dtype(args.precision)
+  save_dtype = str_to_dtype(args.save_precision)
+  if save_dtype is None:
+    save_dtype = merge_dtype
+
+  new_conv_rank = args.new_conv_rank if args.new_conv_rank is not None else args.new_rank
+  state_dict = merge_lora_models(args.models, args.ratios, args.new_rank, new_conv_rank, args.device, merge_dtype)
+
+  print(f"saving model to: {args.save_to}")
+  save_to_file(args.save_to, state_dict, save_dtype)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--save_precision", type=str, default=None,
+                      choices=[None, "float", "fp16", "bf16"], help="precision in saving, same to merging if omitted / 保存時に精度を変更して保存する、省略時はマージ時の精度と同じ")
+  parser.add_argument("--precision", type=str, default="float",
+                      choices=["float", "fp16", "bf16"], help="precision in merging (float is recommended) / マージの計算時の精度（floatを推奨）")
+  parser.add_argument("--save_to", type=str, default=None,
+                      help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors")
+  parser.add_argument("--models", type=str, nargs='*',
+                      help="LoRA models to merge: ckpt or safetensors file / マージするLoRAモデル、ckptまたはsafetensors")
+  parser.add_argument("--ratios", type=float, nargs='*',
+                      help="ratios for each model / それぞれのLoRAモデルの比率")
+  parser.add_argument("--new_rank", type=int, default=4,
+                      help="Specify rank of output LoRA / 出力するLoRAのrank (dim)")
+  parser.add_argument("--new_conv_rank", type=int, default=None,
+                      help="Specify rank of output LoRA for Conv2d 3x3, None for same as new_rank / 出力するConv2D 3x3 LoRAのrank (dim)、Noneでnew_rankと同じ")
+  parser.add_argument("--device", type=str, default=None, help="device to use, cuda for GPU / 計算を行うデバイス、cuda でGPUを使う")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+  merge(args)
diff --git a/presets/finetune/adafactor.json b/presets/finetune/adafactor.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e0149dc71266611a198625230ba90260feffa15
--- /dev/null
+++ b/presets/finetune/adafactor.json
@@ -0,0 +1,61 @@
+{
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "v2": false,
+  "v_parameterization": false,
+  "train_dir": "D:/dataset/paige_spiranac/ft",
+  "image_folder": "D:\\dataset\\paige_spiranac\\lora\\img4_g8\\16_paige_spiranac",
+  "output_dir": "D:/models/test",
+  "logging_dir": "D:/dataset/paige_spiranac/ft/logs",
+  "max_resolution": "512,512",
+  "min_bucket_reso": "256",
+  "max_bucket_reso": "1024",
+  "batch_size": "1",
+  "flip_aug": false,
+  "caption_metadata_filename": "meta_cap.json",
+  "latent_metadata_filename": "meta_lat.json",
+  "full_path": true,
+  "learning_rate": "1e-6",
+  "lr_scheduler": "adafactor",
+  "lr_warmup": "10",
+  "dataset_repeats": "10",
+  "train_batch_size": 4,
+  "epoch": "2",
+  "save_every_n_epochs": "1",
+  "mixed_precision": "bf16",
+  "save_precision": "fp16",
+  "seed": "1234",
+  "num_cpu_threads_per_process": 2,
+  "train_text_encoder": true,
+  "create_caption": true,
+  "create_buckets": false,
+  "save_model_as": "safetensors",
+  "caption_extension": ".txt",
+  "use_8bit_adam": false,
+  "xformers": true,
+  "clip_skip": 1,
+  "save_state": false,
+  "resume": "",
+  "gradient_checkpointing": false,
+  "gradient_accumulation_steps": 1.0,
+  "mem_eff_attn": false,
+  "shuffle_caption": true,
+  "output_name": "paige_spiranac_v1.5e",
+  "max_token_length": "150",
+  "max_train_epochs": "",
+  "max_data_loader_n_workers": "0",
+  "full_fp16": false,
+  "color_aug": false,
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "cache_latents": true,
+  "use_latent_files": "No",
+  "keep_tokens": 1,
+  "persistent_data_loader_workers": false,
+  "bucket_no_upscale": true,
+  "random_crop": false,
+  "bucket_reso_steps": 1.0,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.1,
+  "optimizer": "Adafactor",
+  "optimizer_args": "scale_parameter=True relative_step=True warmup_init=True weight_decay=2",
+  "noise_offset": ""
+}
\ No newline at end of file
diff --git a/presets/finetune/lion.json b/presets/finetune/lion.json
new file mode 100644
index 0000000000000000000000000000000000000000..982c8a869f807acdfec0e936eeef22e19fe093d9
--- /dev/null
+++ b/presets/finetune/lion.json
@@ -0,0 +1,61 @@
+{
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "v2": false,
+  "v_parameterization": false,
+  "train_dir": "D:/dataset/paige_spiranac/ft",
+  "image_folder": "D:\\dataset\\paige_spiranac\\lora\\img4_g8\\16_paige_spiranac",
+  "output_dir": "D:/models/test",
+  "logging_dir": "D:/dataset/paige_spiranac/ft/logs",
+  "max_resolution": "512,512",
+  "min_bucket_reso": "256",
+  "max_bucket_reso": "1024",
+  "batch_size": "1",
+  "flip_aug": false,
+  "caption_metadata_filename": "meta_cap.json",
+  "latent_metadata_filename": "meta_lat.json",
+  "full_path": true,
+  "learning_rate": "0.0000166666666",
+  "lr_scheduler": "cosine",
+  "lr_warmup": "10",
+  "dataset_repeats": "10",
+  "train_batch_size": 4,
+  "epoch": "2",
+  "save_every_n_epochs": "1",
+  "mixed_precision": "bf16",
+  "save_precision": "fp16",
+  "seed": "1234",
+  "num_cpu_threads_per_process": 2,
+  "train_text_encoder": true,
+  "create_caption": true,
+  "create_buckets": false,
+  "save_model_as": "safetensors",
+  "caption_extension": ".txt",
+  "use_8bit_adam": false,
+  "xformers": true,
+  "clip_skip": 1,
+  "save_state": false,
+  "resume": "",
+  "gradient_checkpointing": false,
+  "gradient_accumulation_steps": 1.0,
+  "mem_eff_attn": false,
+  "shuffle_caption": true,
+  "output_name": "paige_spiranac_v1.5e",
+  "max_token_length": "150",
+  "max_train_epochs": "",
+  "max_data_loader_n_workers": "0",
+  "full_fp16": false,
+  "color_aug": false,
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "cache_latents": true,
+  "use_latent_files": "No",
+  "keep_tokens": 1,
+  "persistent_data_loader_workers": false,
+  "bucket_no_upscale": true,
+  "random_crop": false,
+  "bucket_reso_steps": 1.0,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.1,
+  "optimizer": "Lion",
+  "optimizer_args": "",
+  "noise_offset": ""
+}
\ No newline at end of file
diff --git a/presets/finetune/user_presets/.put your own presets here b/presets/finetune/user_presets/.put your own presets here
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/presets/lora/iA3-Prodigy-sd15.json b/presets/lora/iA3-Prodigy-sd15.json
new file mode 100644
index 0000000000000000000000000000000000000000..4a4a2b256cc0c8ce10cd013aa5c2ab62d1acba73
--- /dev/null
+++ b/presets/lora/iA3-Prodigy-sd15.json
@@ -0,0 +1,30 @@
+{
+    "LoRA_type": "LyCORIS/iA3",
+    "adaptive_noise_scale": 0.005,
+    "caption_dropout_rate": 0.5,
+    "epoch": 300,
+    "gradient_accumulation_steps": 1,
+    "gradient_checkpointing": true,
+    "keep_tokens": 1,
+    "learning_rate": 1.0,
+    "lr_scheduler": "constant",
+    "lr_warmup": 0,
+    "min_snr_gamma": 5,
+    "network_alpha": 1024,
+    "network_dim": 1024,
+    "network_dropout": 0.3,
+    "noise_offset": 0.05,
+    "noise_offset_type": "Original",
+    "optimizer": "Prodigy",
+    "optimizer_args": "d_coef=1.0 weight_decay=0.01 safeguard_warmup=False use_bias_correction=False",
+    "save_every_n_epochs": 10,
+    "save_every_n_steps": 0,
+    "save_last_n_steps": 0,
+    "scale_weight_norms": 1,
+    "seed": "31337",
+    "shuffle_caption": true,
+    "text_encoder_lr": 1.0,
+    "train_batch_size": 1,
+    "training_comment": "rentry.co/ProdiAgy",
+    "unet_lr": 1.0
+}
\ No newline at end of file
diff --git a/presets/lora/ia3-sd15.json b/presets/lora/ia3-sd15.json
new file mode 100644
index 0000000000000000000000000000000000000000..3cd8ae08b344f33a8dd9fade9057ecaac6855b99
--- /dev/null
+++ b/presets/lora/ia3-sd15.json
@@ -0,0 +1,87 @@
+{
+    "LoRA_type": "LyCORIS/iA3",
+    "adaptive_noise_scale": 0,
+    "additional_parameters": "",
+    "block_alphas": "",
+    "block_dims": "",
+    "block_lr_zero_threshold": "",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 1,
+    "cache_latents": true,
+    "cache_latents_to_disk": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".none-use-foldername",
+    "clip_skip": "1",
+    "color_aug": false,
+    "conv_alpha": 64,
+    "conv_alphas": "",
+    "conv_dim": 64,
+    "conv_dims": "",
+    "decompose_both": false,
+    "dim_from_weights": false,
+    "down_lr_weight": "",
+    "enable_bucket": true,
+    "epoch": 4,
+    "factor": -1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 1,
+    "gradient_checkpointing": false,
+    "keep_tokens": "0",
+    "learning_rate": 1.0,
+    "lora_network_weights": "",
+    "lr_scheduler": "cosine",
+    "lr_scheduler_num_cycles": "",
+    "lr_scheduler_power": "",
+    "lr_warmup": 0,
+    "max_data_loader_n_workers": "0",
+    "max_resolution": "512,512",
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": false,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 10,
+    "mixed_precision": "bf16",
+    "module_dropout": 0,
+    "multires_noise_discount": 0.2,
+    "multires_noise_iterations": 8,
+    "network_alpha": 64,
+    "network_dim": 64,
+    "network_dropout": 0,
+    "no_token_padding": false,
+    "noise_offset": 0,
+    "noise_offset_type": "Multires",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "Prodigy",
+    "optimizer_args": "",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "rank_dropout": 0,
+    "save_every_n_epochs": 1,
+    "save_every_n_steps": 0,
+    "save_last_n_steps": 0,
+    "save_last_n_steps_state": 0,
+    "save_precision": "fp16",
+    "scale_v_pred_loss_like_noise_pred": false,
+    "scale_weight_norms": 0,
+    "seed": "",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": 1.0,
+    "train_batch_size": 1,
+    "train_on_input": true,
+    "training_comment": "",
+    "unet_lr": 1.0,
+    "unit": 1,
+    "up_lr_weight": "",
+    "use_cp": false,
+    "use_wandb": false,
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "wandb_api_key": "",
+    "weighted_captions": false,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/locon-dadaptation-sdxl.json b/presets/lora/locon-dadaptation-sdxl.json
new file mode 100644
index 0000000000000000000000000000000000000000..8c643d2df6a05ff61ff0b12289976ac41bf86b10
--- /dev/null
+++ b/presets/lora/locon-dadaptation-sdxl.json
@@ -0,0 +1,88 @@
+{
+    "LoRA_type": "Standard",
+    "adaptive_noise_scale": 0,
+    "additional_parameters": "",
+    "block_alphas": "",
+    "block_dims": "",
+    "block_lr_zero_threshold": "",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 1,
+    "cache_latents": true,
+    "cache_latents_to_disk": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".none-use-foldername",
+    "clip_skip": "1",
+    "color_aug": false,
+    "conv_alpha": 64,
+    "conv_alphas": "",
+    "conv_dim": 64,
+    "conv_dims": "",
+    "decompose_both": false,
+    "dim_from_weights": false,
+    "down_lr_weight": "",
+    "enable_bucket": true,
+    "epoch": 4,
+    "factor": -1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 1,
+    "gradient_checkpointing": false,
+    "keep_tokens": "0",
+    "learning_rate": 4e-07,
+    "lora_network_weights": "",
+    "lr_scheduler": "constant_with_warmup",
+    "lr_scheduler_num_cycles": "",
+    "lr_scheduler_power": "",
+    "lr_warmup": 8,
+    "max_data_loader_n_workers": "0",
+    "max_resolution": "512,512",
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": false,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 10,
+    "mixed_precision": "bf16",
+    "module_dropout": 0,
+    "multires_noise_discount": 0.2,
+    "multires_noise_iterations": 8,
+    "network_alpha": 64,
+    "network_dim": 64,
+    "network_dropout": 0,
+    "no_token_padding": false,
+    "noise_offset": 0.0357,
+    "noise_offset_type": "Original",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "Adafactor",
+    "optimizer_args": "scale_parameter=False relative_step=False warmup_init=False",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "rank_dropout": 0,
+    "save_every_n_epochs": 1,
+    "save_every_n_steps": 0,
+    "save_last_n_steps": 0,
+    "save_last_n_steps_state": 0,
+    "save_precision": "fp16",
+    "scale_v_pred_loss_like_noise_pred": false,
+    "scale_weight_norms": 0,
+    "sdxl": true,
+    "seed": "",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": 0.0,
+    "train_batch_size": 1,
+    "train_on_input": true,
+    "training_comment": "",
+    "unet_lr": 4e-07,
+    "unit": 1,
+    "up_lr_weight": "",
+    "use_cp": false,
+    "use_wandb": false,
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "wandb_api_key": "",
+    "weighted_captions": false,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/loha-sd15.json b/presets/lora/loha-sd15.json
new file mode 100644
index 0000000000000000000000000000000000000000..b66bf28a30ac013ca0459c63811dc3cbc8688c0f
--- /dev/null
+++ b/presets/lora/loha-sd15.json
@@ -0,0 +1,87 @@
+{
+    "LoRA_type": "LyCORIS/LoHa",
+    "adaptive_noise_scale": 0,
+    "additional_parameters": "",
+    "block_alphas": "",
+    "block_dims": "",
+    "block_lr_zero_threshold": "",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 1,
+    "cache_latents": true,
+    "cache_latents_to_disk": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".none-use-foldername",
+    "clip_skip": "1",
+    "color_aug": false,
+    "conv_alpha": 4,
+    "conv_alphas": "",
+    "conv_dim": 8,
+    "conv_dims": "",
+    "decompose_both": false,
+    "dim_from_weights": false,
+    "down_lr_weight": "",
+    "enable_bucket": true,
+    "epoch": 2,
+    "factor": -1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 4,
+    "gradient_checkpointing": false,
+    "keep_tokens": "0",
+    "learning_rate": 0.0001,
+    "lora_network_weights": "",
+    "lr_scheduler": "cosine",
+    "lr_scheduler_num_cycles": "",
+    "lr_scheduler_power": "",
+    "lr_warmup": 0,
+    "max_data_loader_n_workers": "0",
+    "max_resolution": "512,512",
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": false,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 10,
+    "mixed_precision": "bf16",
+    "module_dropout": 0,
+    "multires_noise_discount": 0.2,
+    "multires_noise_iterations": 8,
+    "network_alpha": 16,
+    "network_dim": 32,
+    "network_dropout": 0,
+    "no_token_padding": false,
+    "noise_offset": 0,
+    "noise_offset_type": "Multires",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "AdamW",
+    "optimizer_args": "",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "rank_dropout": 0,
+    "save_every_n_epochs": 1,
+    "save_every_n_steps": 0,
+    "save_last_n_steps": 0,
+    "save_last_n_steps_state": 0,
+    "save_precision": "fp16",
+    "scale_v_pred_loss_like_noise_pred": false,
+    "scale_weight_norms": 1,
+    "seed": "",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": 0.0001,
+    "train_batch_size": 1,
+    "train_on_input": true,
+    "training_comment": "",
+    "unet_lr": 0.0001,
+    "unit": 1,
+    "up_lr_weight": "",
+    "use_cp": true,
+    "use_wandb": false,
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "wandb_api_key": "",
+    "weighted_captions": false,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/lokr-sd15.json b/presets/lora/lokr-sd15.json
new file mode 100644
index 0000000000000000000000000000000000000000..8f1cdc8cb08a2c88302aecc6c92cc62f3fe43f84
--- /dev/null
+++ b/presets/lora/lokr-sd15.json
@@ -0,0 +1,85 @@
+{
+    "LoRA_type": "LyCORIS/LoKr",
+    "adaptive_noise_scale": 0,
+    "additional_parameters": "",
+    "block_alphas": "",
+    "block_dims": "",
+    "block_lr_zero_threshold": "",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 1,
+    "cache_latents": true,
+    "cache_latents_to_disk": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".none-use-foldername",
+    "clip_skip": "1",
+    "color_aug": false,
+    "conv_alpha": 64,
+    "conv_alphas": "",
+    "conv_dim": 64,
+    "conv_dims": "",
+    "decompose_both": false,
+    "dim_from_weights": false,
+    "down_lr_weight": "",
+    "enable_bucket": true,
+    "factor": -1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 4,
+    "gradient_checkpointing": false,
+    "keep_tokens": "0",
+    "learning_rate": 1.0,
+    "lora_network_weights": "",
+    "lr_scheduler": "cosine",
+    "lr_scheduler_num_cycles": "",
+    "lr_scheduler_power": "",
+    "lr_warmup": 0,
+    "max_data_loader_n_workers": "0",
+    "max_resolution": "512,512",
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": false,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 10,
+    "mixed_precision": "bf16",
+    "module_dropout": 0,
+    "multires_noise_discount": 0.2,
+    "multires_noise_iterations": 8,
+    "network_alpha": 64,
+    "network_dim": 64,
+    "network_dropout": 0,
+    "no_token_padding": false,
+    "noise_offset": 0,
+    "noise_offset_type": "Multires",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "Prodigy",
+    "optimizer_args": "",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "rank_dropout": 0,
+    "save_every_n_steps": 0,
+    "save_last_n_steps": 0,
+    "save_last_n_steps_state": 0,
+    "save_precision": "fp16",
+    "scale_v_pred_loss_like_noise_pred": false,
+    "scale_weight_norms": 0,
+    "seed": "",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": 1.0,
+    "train_batch_size": 1,
+    "train_on_input": false,
+    "training_comment": "",
+    "unet_lr": 1.0,
+    "unit": 1,
+    "up_lr_weight": "",
+    "use_cp": false,
+    "use_wandb": false,
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "wandb_api_key": "",
+    "weighted_captions": false,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/prepare_presets.md b/presets/lora/prepare_presets.md
new file mode 100644
index 0000000000000000000000000000000000000000..bf8c900696b62bb62cf6dc732d38d781afa688c6
--- /dev/null
+++ b/presets/lora/prepare_presets.md
@@ -0,0 +1,5 @@
+# Preparing presets for users
+
+Run the followinf command to prepare new presets for release to users:
+
+`python.exe .\tools\prepare_presets.py .\presets\lora\*.json`
\ No newline at end of file
diff --git a/presets/lora/sd15-EDG_LoConOptiSettings.json b/presets/lora/sd15-EDG_LoConOptiSettings.json
new file mode 100644
index 0000000000000000000000000000000000000000..88057eb33ac656ab66b2a36877d7aac6d43cede1
--- /dev/null
+++ b/presets/lora/sd15-EDG_LoConOptiSettings.json
@@ -0,0 +1,65 @@
+{
+    "LoRA_type": "LyCORIS/LoCon",
+    "additional_parameters": "",
+    "block_alphas": "",
+    "block_dims": "",
+    "block_lr_zero_threshold": "",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 64.0,
+    "cache_latents": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".txt",
+    "clip_skip": 2,
+    "color_aug": false,
+    "conv_alpha": 1,
+    "conv_alphas": "",
+    "conv_dim": 32,
+    "conv_dims": "",
+    "down_lr_weight": "",
+    "enable_bucket": false,
+    "epoch": 1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 1.0,
+    "gradient_checkpointing": false,
+    "keep_tokens": "0",
+    "learning_rate": "0.0001",
+    "lora_network_weights": "",
+    "lr_scheduler": "constant",
+    "lr_scheduler_num_cycles": "",
+    "lr_scheduler_power": "",
+    "lr_warmup": "0",
+    "max_data_loader_n_workers": "1",
+    "max_resolution": "512,650",
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": true,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 0,
+    "mixed_precision": "bf16",
+    "network_alpha": 64,
+    "network_dim": 64,
+    "no_token_padding": false,
+    "noise_offset": "0.05",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "AdamW8bit",
+    "optimizer_args": "",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "save_every_n_epochs": 1,
+    "save_precision": "bf16",
+    "seed": "1234",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": "5e-05",
+    "train_batch_size": 3,
+    "training_comment": "",
+    "unet_lr": "0.0001",
+    "up_lr_weight": "",
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/sd15-EDG_LoHaOptiSettings.json b/presets/lora/sd15-EDG_LoHaOptiSettings.json
new file mode 100644
index 0000000000000000000000000000000000000000..518b19c867251e8f93b501abe2b0ab8d7bb60bbd
--- /dev/null
+++ b/presets/lora/sd15-EDG_LoHaOptiSettings.json
@@ -0,0 +1,65 @@
+{
+    "LoRA_type": "LyCORIS/LoHa",
+    "additional_parameters": "",
+    "block_alphas": "",
+    "block_dims": "",
+    "block_lr_zero_threshold": "",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 64.0,
+    "cache_latents": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".txt",
+    "clip_skip": 2,
+    "color_aug": false,
+    "conv_alpha": 1,
+    "conv_alphas": "",
+    "conv_dim": 32,
+    "conv_dims": "",
+    "down_lr_weight": "",
+    "enable_bucket": false,
+    "epoch": 1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 1.0,
+    "gradient_checkpointing": false,
+    "keep_tokens": "0",
+    "learning_rate": "0.0001",
+    "lora_network_weights": "",
+    "lr_scheduler": "constant",
+    "lr_scheduler_num_cycles": "1",
+    "lr_scheduler_power": "",
+    "lr_warmup": "0",
+    "max_data_loader_n_workers": "1",
+    "max_resolution": "512,650",
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": true,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 0,
+    "mixed_precision": "bf16",
+    "network_alpha": 32,
+    "network_dim": 32,
+    "no_token_padding": false,
+    "noise_offset": "",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "AdamW8bit",
+    "optimizer_args": "",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "save_every_n_epochs": 1,
+    "save_precision": "bf16",
+    "seed": "1234",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": "5e-5",
+    "train_batch_size": 3,
+    "training_comment": "",
+    "unet_lr": "0.0001",
+    "up_lr_weight": "",
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/sd15-EDG_LoraOptiSettings.json b/presets/lora/sd15-EDG_LoraOptiSettings.json
new file mode 100644
index 0000000000000000000000000000000000000000..e7780315d1e3f4d01e618262fc45ae6d1f2d7a5f
--- /dev/null
+++ b/presets/lora/sd15-EDG_LoraOptiSettings.json
@@ -0,0 +1,65 @@
+{
+    "LoRA_type": "Standard",
+    "additional_parameters": "",
+    "block_alphas": "",
+    "block_dims": "",
+    "block_lr_zero_threshold": "",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 64.0,
+    "cache_latents": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".txt",
+    "clip_skip": 2,
+    "color_aug": false,
+    "conv_alpha": 1,
+    "conv_alphas": "",
+    "conv_dim": 1,
+    "conv_dims": "",
+    "down_lr_weight": "",
+    "enable_bucket": false,
+    "epoch": 1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 1.0,
+    "gradient_checkpointing": false,
+    "keep_tokens": "0",
+    "learning_rate": "0.0001",
+    "lora_network_weights": "",
+    "lr_scheduler": "constant",
+    "lr_scheduler_num_cycles": "",
+    "lr_scheduler_power": "",
+    "lr_warmup": "0",
+    "max_data_loader_n_workers": "1",
+    "max_resolution": "512,650",
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": true,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 0,
+    "mixed_precision": "bf16",
+    "network_alpha": 64,
+    "network_dim": 64,
+    "no_token_padding": false,
+    "noise_offset": "0.05",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "AdamW8bit",
+    "optimizer_args": "",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "save_every_n_epochs": 1,
+    "save_precision": "bf16",
+    "seed": "1234",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": "5e-05",
+    "train_batch_size": 3,
+    "training_comment": "",
+    "unet_lr": "0.0001",
+    "up_lr_weight": "",
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/sdxl-24GB-loha-prodigy_v1.json b/presets/lora/sdxl-24GB-loha-prodigy_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..5cb0da38921ee04c4cbf1815da451d64d7bf4f59
--- /dev/null
+++ b/presets/lora/sdxl-24GB-loha-prodigy_v1.json
@@ -0,0 +1,84 @@
+{
+    "LoRA_type": "LyCORIS/LoHa",
+    "adaptive_noise_scale": 0,
+    "additional_parameters": "--max_grad_norm=0",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 64,
+    "cache_latents": true,
+    "cache_latents_to_disk": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".none-use-foldername",
+    "clip_skip": "1",
+    "color_aug": false,
+    "conv_alpha": 4,
+    "conv_alphas": "",
+    "conv_dim": 8,
+    "conv_dims": "",
+    "decompose_both": false,
+    "dim_from_weights": false,
+    "down_lr_weight": "",
+    "enable_bucket": true,
+    "epoch": 20,
+    "factor": -1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 1.0,
+    "gradient_checkpointing": true,
+    "keep_tokens": "0",
+    "learning_rate": 1,
+    "lora_network_weights": "",
+    "lr_scheduler": "cosine",
+    "lr_scheduler_num_cycles": "",
+    "lr_scheduler_power": "",
+    "lr_warmup": 0,
+    "max_data_loader_n_workers": "0",
+    "max_resolution": "1024,1024",
+    "max_timestep": 1000,
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": false,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 0,
+    "min_timestep": 0,
+    "mixed_precision": "fp16",
+    "module_dropout": 0,
+    "multires_noise_discount": 0.2,
+    "multires_noise_iterations": 8,
+    "network_alpha": 16,
+    "network_dim": 32,
+    "network_dropout": 0,
+    "no_token_padding": false,
+    "noise_offset": 0.0357,
+    "noise_offset_type": "Original",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "Prodigy",
+    "optimizer_args": "decouple=True weight_decay=0.5 betas=0.9,0.99 use_bias_correction=False",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "rank_dropout": 0,
+    "save_every_n_epochs": 2,
+    "save_every_n_steps": 0,
+    "save_last_n_steps": 0,
+    "save_last_n_steps_state": 0,
+    "save_precision": "fp16",
+    "scale_v_pred_loss_like_noise_pred": false,
+    "scale_weight_norms": 1,
+    "sdxl_cache_text_encoder_outputs": false,
+    "sdxl_no_half_vae": false,
+    "seed": "12345",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": 1,
+    "train_batch_size": 8,
+    "unet_lr": 1,
+    "unit": 1,
+    "up_lr_weight": "",
+    "use_cp": true,
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "weighted_captions": false,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/sdxl-24GB-lokr_v1.json b/presets/lora/sdxl-24GB-lokr_v1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5768ee483769a22eedf07ba2e07a113cb79cb0c
--- /dev/null
+++ b/presets/lora/sdxl-24GB-lokr_v1.json
@@ -0,0 +1,92 @@
+{
+    "LoRA_type": "LyCORIS/LoKr",
+    "adaptive_noise_scale": 0,
+    "additional_parameters": "",
+    "block_alphas": "",
+    "block_dims": "",
+    "block_lr_zero_threshold": "",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 64,
+    "cache_latents": true,
+    "cache_latents_to_disk": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".txt",
+    "clip_skip": "1",
+    "color_aug": false,
+    "conv_alpha": 64,
+    "conv_alphas": "",
+    "conv_dim": 64,
+    "conv_dims": "",
+    "decompose_both": false,
+    "dim_from_weights": false,
+    "down_lr_weight": "",
+    "enable_bucket": true,
+    "epoch": 20,
+    "factor": -1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 1.0,
+    "gradient_checkpointing": true,
+    "keep_tokens": "0",
+    "learning_rate": 1.0,
+    "lora_network_weights": "",
+    "lr_scheduler": "constant",
+    "lr_scheduler_num_cycles": "",
+    "lr_scheduler_power": "",
+    "lr_warmup": 0,
+    "max_data_loader_n_workers": "0",
+    "max_resolution": "1024,1024",
+    "max_timestep": 1000,
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": false,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 10,
+    "min_timestep": 0,
+    "mixed_precision": "bf16",
+    "module_dropout": 0.1,
+    "multires_noise_discount": 0.2,
+    "multires_noise_iterations": 8,
+    "network_alpha": 64,
+    "network_dim": 64,
+    "network_dropout": 0,
+    "no_token_padding": false,
+    "noise_offset": 0.0357,
+    "noise_offset_type": "Multires",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "Prodigy",
+    "optimizer_args": "",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "rank_dropout": 0.1,
+    "save_every_n_epochs": 1,
+    "save_every_n_steps": 0,
+    "save_last_n_steps": 0,
+    "save_last_n_steps_state": 0,
+    "save_precision": "fp16",
+    "scale_v_pred_loss_like_noise_pred": false,
+    "scale_weight_norms": 0,
+    "sdxl": true,
+    "sdxl_cache_text_encoder_outputs": false,
+    "sdxl_no_half_vae": true,
+    "seed": "12345",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": 1.0,
+    "train_batch_size": 8,
+    "train_on_input": true,
+    "training_comment": "",
+    "unet_lr": 1.0,
+    "unit": 1,
+    "up_lr_weight": "",
+    "use_cp": true,
+    "use_wandb": false,
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "wandb_api_key": "",
+    "weighted_captions": false,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/sdxl-loha-AdamW8bit-kBlueLeafv1.json b/presets/lora/sdxl-loha-AdamW8bit-kBlueLeafv1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec3997ba0e49247d51baf5d4748b9713cb3a2be9
--- /dev/null
+++ b/presets/lora/sdxl-loha-AdamW8bit-kBlueLeafv1.json
@@ -0,0 +1,92 @@
+{
+    "LoRA_type": "LyCORIS/LoHa",
+    "adaptive_noise_scale": 0,
+    "additional_parameters": "--min_bucket_reso=512 --max_bucket_reso=2048 --log_prefix=xl-loha",
+    "block_alphas": "",
+    "block_dims": "",
+    "block_lr_zero_threshold": "",
+    "bucket_no_upscale": true,
+    "bucket_reso_steps": 64,
+    "cache_latents": true,
+    "cache_latents_to_disk": true,
+    "caption_dropout_every_n_epochs": 0.0,
+    "caption_dropout_rate": 0,
+    "caption_extension": ".txt",
+    "clip_skip": "1",
+    "color_aug": false,
+    "conv_alpha": 4,
+    "conv_alphas": "",
+    "conv_dim": 4,
+    "conv_dims": "",
+    "decompose_both": false,
+    "dim_from_weights": false,
+    "down_lr_weight": "",
+    "enable_bucket": true,
+    "epoch": 10,
+    "factor": -1,
+    "flip_aug": false,
+    "full_fp16": false,
+    "gradient_accumulation_steps": 1.0,
+    "gradient_checkpointing": true,
+    "keep_tokens": "0",
+    "learning_rate": 0.002,
+    "lora_network_weights": "",
+    "lr_scheduler": "cosine",
+    "lr_scheduler_num_cycles": "1",
+    "lr_scheduler_power": "",
+    "lr_warmup": 0,
+    "max_data_loader_n_workers": "0",
+    "max_resolution": "1024,1024",
+    "max_timestep": 1000,
+    "max_token_length": "75",
+    "max_train_epochs": "",
+    "mem_eff_attn": false,
+    "mid_lr_weight": "",
+    "min_snr_gamma": 5,
+    "min_timestep": 0,
+    "mixed_precision": "bf16",
+    "module_dropout": 0,
+    "multires_noise_discount": 0,
+    "multires_noise_iterations": 0,
+    "network_alpha": 8,
+    "network_dim": 8,
+    "network_dropout": 0,
+    "no_token_padding": false,
+    "noise_offset": 0.0357,
+    "noise_offset_type": "Original",
+    "num_cpu_threads_per_process": 2,
+    "optimizer": "AdamW8bit",
+    "optimizer_args": "weight_decay=0.05 betas=0.9,0.98",
+    "persistent_data_loader_workers": false,
+    "prior_loss_weight": 1.0,
+    "random_crop": false,
+    "rank_dropout": 0,
+    "save_every_n_epochs": 1,
+    "save_every_n_steps": 0,
+    "save_last_n_steps": 0,
+    "save_last_n_steps_state": 0,
+    "save_precision": "bf16",
+    "scale_v_pred_loss_like_noise_pred": false,
+    "scale_weight_norms": 0,
+    "sdxl": true,
+    "sdxl_cache_text_encoder_outputs": true,
+    "sdxl_no_half_vae": true,
+    "seed": "17415",
+    "shuffle_caption": false,
+    "stop_text_encoder_training": 0,
+    "text_encoder_lr": 0.0,
+    "train_batch_size": 4,
+    "train_on_input": false,
+    "training_comment": "",
+    "unet_lr": 0.002,
+    "unit": 1,
+    "up_lr_weight": "",
+    "use_cp": false,
+    "use_wandb": false,
+    "v2": false,
+    "v_parameterization": false,
+    "vae_batch_size": 0,
+    "wandb_api_key": "",
+    "weighted_captions": false,
+    "xformers": true
+}
\ No newline at end of file
diff --git a/presets/lora/user_presets/.put your own presets here b/presets/lora/user_presets/.put your own presets here
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000000000000000000000000000000000000..83201610c950394be4c9246f48dba1efbfa6f3bf
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,19 @@
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "library"
+version = "1.0.3"
+description = "Libraries required to run kohya_ss GUI"
+authors = ["Bernard Maltais <bernard@ducourier.com>"]
+license = "Apache-2.0" # Apache Software License
+
+[[tool.poetry.source]]
+name = "library"
+path = "library"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.11"
+
+[tool.poetry.dev-dependencies]
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..cbaf39c023cdcd8c37fc7629de31be3c8691b234
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,27 @@
+albumentations==1.3.0
+altair==4.2.2
+dadaptation==3.1
+diffusers[torch]==0.17.1
+easygui==0.98.3
+einops==0.6.0
+fairscale==0.4.13
+ftfy==6.1.1
+gradio==3.36.1
+huggingface-hub>=0.14.1
+lion-pytorch==0.0.6
+lycoris_lora==1.8.0.dev6
+# lycoris_lora==1.7.2
+open-clip-torch==2.20.0
+opencv-python==4.7.0.68
+prodigyopt==1.0
+pytorch-lightning==1.9.0
+rich==13.4.1
+safetensors==0.3.1
+timm==0.6.12
+tk==0.1.0
+toml==0.10.2
+transformers==4.30.2
+voluptuous==0.13.1
+wandb==0.15.0
+# for kohya_ss library
+-e . # no_verify leave this to specify not checking this a verification stage
diff --git a/requirements_linux.txt b/requirements_linux.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e7177d6ac411d56de110aeec1d7c20e80a8a8185
--- /dev/null
+++ b/requirements_linux.txt
@@ -0,0 +1,4 @@
+torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 # no_verify leave this to specify not checking this a verification stage
+xformers==0.0.20 bitsandbytes==0.35.0
+accelerate==0.19.0 tensorboard==2.12.1 tensorflow==2.12.0
+-r requirements.txt
diff --git a/requirements_linux_docker.txt b/requirements_linux_docker.txt
new file mode 100644
index 0000000000000000000000000000000000000000..25a90c2b17fb1ba8eacbea08ee662a41a0398c7b
--- /dev/null
+++ b/requirements_linux_docker.txt
@@ -0,0 +1,5 @@
+xformers==0.0.20
+bitsandbytes==0.35.0
+accelerate==0.19.0
+tensorboard==2.12.1
+tensorflow==2.12.0
diff --git a/requirements_macos_amd64.txt b/requirements_macos_amd64.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6aff7e4051420a2be73ff73f69d15b3ca8e72225
--- /dev/null
+++ b/requirements_macos_amd64.txt
@@ -0,0 +1,4 @@
+torch==2.0.0 torchvision==0.15.1 -f https://download.pytorch.org/whl/cpu/torch_stable.html
+xformers bitsandbytes==0.35.0
+accelerate==0.19.0 tensorflow-macos tensorboard==2.12.1
+-r requirements.txt
diff --git a/requirements_macos_arm64.txt b/requirements_macos_arm64.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d1bc2b3a33c97a16c736333b5fb0143eb4bb4b1b
--- /dev/null
+++ b/requirements_macos_arm64.txt
@@ -0,0 +1,4 @@
+torch==2.0.0 torchvision==0.15.1 -f https://download.pytorch.org/whl/cpu/torch_stable.html
+xformers bitsandbytes==0.35.0
+accelerate==0.19.0 tensorflow-metal tensorboard==2.12.1
+-r requirements.txt
diff --git a/requirements_runpod.txt b/requirements_runpod.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8d69fad4748fa9da93701c8cf1557d3cf0aff5af
--- /dev/null
+++ b/requirements_runpod.txt
@@ -0,0 +1,5 @@
+torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 # no_verify leave this to specify not checking this a verification stage
+xformers==0.0.20 bitsandbytes==0.35.0
+accelerate==0.19.0 tensorboard==2.12.1 tensorflow==2.12.0 wheel
+tensorrt
+-r requirements.txt
diff --git a/requirements_windows_torch1.txt b/requirements_windows_torch1.txt
new file mode 100644
index 0000000000000000000000000000000000000000..81d5d49e61a7c5b1cc1f6823ace45b6094c0e211
--- /dev/null
+++ b/requirements_windows_torch1.txt
@@ -0,0 +1,5 @@
+torch==1.12.1+cu116 torchvision==0.13.1+cu116 --index-url https://download.pytorch.org/whl/cu116 # no_verify
+https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl -U -I --no-deps # no_verify
+bitsandbytes==0.35.0
+accelerate==0.19.0 tensorboard==2.10.1 tensorflow==2.10.1
+-r requirements.txt
diff --git a/requirements_windows_torch2.txt b/requirements_windows_torch2.txt
new file mode 100644
index 0000000000000000000000000000000000000000..34ae0eabf17c4a3aab34ba616be2b4abf6ac0d36
--- /dev/null
+++ b/requirements_windows_torch2.txt
@@ -0,0 +1,4 @@
+torch==2.0.1+cu118 torchvision==0.15.2+cu118 --index-url https://download.pytorch.org/whl/cu118 # no_verify
+xformers==0.0.20 bitsandbytes==0.35.0
+accelerate==0.19.0 tensorboard==2.12.3 tensorflow==2.12.0
+-r requirements.txt
diff --git a/sdxl_gen_img.py b/sdxl_gen_img.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e20595cc78fa06012ae4696f18cfc7db8443cc6
--- /dev/null
+++ b/sdxl_gen_img.py
@@ -0,0 +1,2558 @@
+import itertools
+import json
+from typing import Any, List, NamedTuple, Optional, Tuple, Union, Callable
+import glob
+import importlib
+import inspect
+import time
+import zipfile
+from diffusers.utils import deprecate
+from diffusers.configuration_utils import FrozenDict
+import argparse
+import math
+import os
+import random
+import re
+
+import diffusers
+import numpy as np
+import torch
+import torchvision
+from diffusers import (
+    AutoencoderKL,
+    DDPMScheduler,
+    EulerAncestralDiscreteScheduler,
+    DPMSolverMultistepScheduler,
+    DPMSolverSinglestepScheduler,
+    LMSDiscreteScheduler,
+    PNDMScheduler,
+    DDIMScheduler,
+    EulerDiscreteScheduler,
+    HeunDiscreteScheduler,
+    KDPM2DiscreteScheduler,
+    KDPM2AncestralDiscreteScheduler,
+    # UNet2DConditionModel,
+    StableDiffusionPipeline,
+)
+from einops import rearrange
+from tqdm import tqdm
+from torchvision import transforms
+from transformers import CLIPTextModel, CLIPTokenizer, CLIPModel, CLIPTextConfig
+import PIL
+from PIL import Image
+from PIL.PngImagePlugin import PngInfo
+
+import library.model_util as model_util
+import library.train_util as train_util
+import library.sdxl_model_util as sdxl_model_util
+import library.sdxl_train_util as sdxl_train_util
+from networks.lora import LoRANetwork
+import tools.original_control_net as original_control_net
+from tools.original_control_net import ControlNetInfo
+from library.sdxl_original_unet import SdxlUNet2DConditionModel
+from library.original_unet import FlashAttentionFunction
+
+# scheduler:
+SCHEDULER_LINEAR_START = 0.00085
+SCHEDULER_LINEAR_END = 0.0120
+SCHEDULER_TIMESTEPS = 1000
+SCHEDLER_SCHEDULE = "scaled_linear"
+
+# その他の設定
+LATENT_CHANNELS = 4
+DOWNSAMPLING_FACTOR = 8
+
+# region モジュール入れ替え部
+"""
+高速化のためのモジュール入れ替え
+"""
+
+
+def replace_unet_modules(unet: diffusers.models.unet_2d_condition.UNet2DConditionModel, mem_eff_attn, xformers, sdpa):
+    if mem_eff_attn:
+        print("Enable memory efficient attention for U-Net")
+
+        # これはDiffusersのU-Netではなく自前のU-Netなので置き換えなくても良い
+        unet.set_use_memory_efficient_attention(False, True)
+    elif xformers:
+        print("Enable xformers for U-Net")
+        try:
+            import xformers.ops
+        except ImportError:
+            raise ImportError("No xformers / xformersがインストールされていないようです")
+
+        unet.set_use_memory_efficient_attention(True, False)
+    elif sdpa:
+        print("Enable SDPA for U-Net")
+        unet.set_use_memory_efficient_attention(False, False)
+        unet.set_use_sdpa(True)
+
+
+# TODO common train_util.py
+def replace_vae_modules(vae: diffusers.models.AutoencoderKL, mem_eff_attn, xformers, sdpa):
+    if mem_eff_attn:
+        replace_vae_attn_to_memory_efficient()
+    elif xformers:
+        replace_vae_attn_to_xformers()
+    elif sdpa:
+        replace_vae_attn_to_sdpa()
+
+
+def replace_vae_attn_to_memory_efficient():
+    print("VAE Attention.forward has been replaced to FlashAttention (not xformers)")
+    flash_func = FlashAttentionFunction
+
+    def forward_flash_attn(self, hidden_states, **kwargs):
+        q_bucket_size = 512
+        k_bucket_size = 1024
+
+        residual = hidden_states
+        batch, channel, height, width = hidden_states.shape
+
+        # norm
+        hidden_states = self.group_norm(hidden_states)
+
+        hidden_states = hidden_states.view(batch, channel, height * width).transpose(1, 2)
+
+        # proj to q, k, v
+        query_proj = self.to_q(hidden_states)
+        key_proj = self.to_k(hidden_states)
+        value_proj = self.to_v(hidden_states)
+
+        query_proj, key_proj, value_proj = map(
+            lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), (query_proj, key_proj, value_proj)
+        )
+
+        out = flash_func.apply(query_proj, key_proj, value_proj, None, False, q_bucket_size, k_bucket_size)
+
+        out = rearrange(out, "b h n d -> b n (h d)")
+
+        # compute next hidden_states
+        # linear proj
+        hidden_states = self.to_out[0](hidden_states)
+        # dropout
+        hidden_states = self.to_out[1](hidden_states)
+
+        hidden_states = hidden_states.transpose(-1, -2).reshape(batch, channel, height, width)
+
+        # res connect and rescale
+        hidden_states = (hidden_states + residual) / self.rescale_output_factor
+        return hidden_states
+
+    def forward_flash_attn_0_14(self, hidden_states, **kwargs):
+        if not hasattr(self, "to_q"):
+            self.to_q = self.query
+            self.to_k = self.key
+            self.to_v = self.value
+            self.to_out = [self.proj_attn, torch.nn.Identity()]
+            self.heads = self.num_heads
+        return forward_flash_attn(self, hidden_states, **kwargs)
+
+    if diffusers.__version__ < "0.15.0":
+        diffusers.models.attention.AttentionBlock.forward = forward_flash_attn_0_14
+    else:
+        diffusers.models.attention_processor.Attention.forward = forward_flash_attn
+
+
+def replace_vae_attn_to_xformers():
+    print("VAE: Attention.forward has been replaced to xformers")
+    import xformers.ops
+
+    def forward_xformers(self, hidden_states, **kwargs):
+        residual = hidden_states
+        batch, channel, height, width = hidden_states.shape
+
+        # norm
+        hidden_states = self.group_norm(hidden_states)
+
+        hidden_states = hidden_states.view(batch, channel, height * width).transpose(1, 2)
+
+        # proj to q, k, v
+        query_proj = self.to_q(hidden_states)
+        key_proj = self.to_k(hidden_states)
+        value_proj = self.to_v(hidden_states)
+
+        query_proj, key_proj, value_proj = map(
+            lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), (query_proj, key_proj, value_proj)
+        )
+
+        query_proj = query_proj.contiguous()
+        key_proj = key_proj.contiguous()
+        value_proj = value_proj.contiguous()
+        out = xformers.ops.memory_efficient_attention(query_proj, key_proj, value_proj, attn_bias=None)
+
+        out = rearrange(out, "b h n d -> b n (h d)")
+
+        # compute next hidden_states
+        # linear proj
+        hidden_states = self.to_out[0](hidden_states)
+        # dropout
+        hidden_states = self.to_out[1](hidden_states)
+
+        hidden_states = hidden_states.transpose(-1, -2).reshape(batch, channel, height, width)
+
+        # res connect and rescale
+        hidden_states = (hidden_states + residual) / self.rescale_output_factor
+        return hidden_states
+
+    def forward_xformers_0_14(self, hidden_states, **kwargs):
+        if not hasattr(self, "to_q"):
+            self.to_q = self.query
+            self.to_k = self.key
+            self.to_v = self.value
+            self.to_out = [self.proj_attn, torch.nn.Identity()]
+            self.heads = self.num_heads
+        return forward_xformers(self, hidden_states, **kwargs)
+
+    if diffusers.__version__ < "0.15.0":
+        diffusers.models.attention.AttentionBlock.forward = forward_xformers_0_14
+    else:
+        diffusers.models.attention_processor.Attention.forward = forward_xformers
+
+
+def replace_vae_attn_to_sdpa():
+    print("VAE: Attention.forward has been replaced to sdpa")
+
+    def forward_sdpa(self, hidden_states, **kwargs):
+        residual = hidden_states
+        batch, channel, height, width = hidden_states.shape
+
+        # norm
+        hidden_states = self.group_norm(hidden_states)
+
+        hidden_states = hidden_states.view(batch, channel, height * width).transpose(1, 2)
+
+        # proj to q, k, v
+        query_proj = self.to_q(hidden_states)
+        key_proj = self.to_k(hidden_states)
+        value_proj = self.to_v(hidden_states)
+
+        query_proj, key_proj, value_proj = map(
+            lambda t: rearrange(t, "b n (h d) -> b n h d", h=self.heads), (query_proj, key_proj, value_proj)
+        )
+
+        out = torch.nn.functional.scaled_dot_product_attention(
+            query_proj, key_proj, value_proj, attn_mask=None, dropout_p=0.0, is_causal=False
+        )
+
+        out = rearrange(out, "b n h d -> b n (h d)")
+
+        # compute next hidden_states
+        # linear proj
+        hidden_states = self.to_out[0](hidden_states)
+        # dropout
+        hidden_states = self.to_out[1](hidden_states)
+
+        hidden_states = hidden_states.transpose(-1, -2).reshape(batch, channel, height, width)
+
+        # res connect and rescale
+        hidden_states = (hidden_states + residual) / self.rescale_output_factor
+        return hidden_states
+
+    def forward_sdpa_0_14(self, hidden_states, **kwargs):
+        if not hasattr(self, "to_q"):
+            self.to_q = self.query
+            self.to_k = self.key
+            self.to_v = self.value
+            self.to_out = [self.proj_attn, torch.nn.Identity()]
+            self.heads = self.num_heads
+        return forward_sdpa(self, hidden_states, **kwargs)
+
+    if diffusers.__version__ < "0.15.0":
+        diffusers.models.attention.AttentionBlock.forward = forward_sdpa_0_14
+    else:
+        diffusers.models.attention_processor.Attention.forward = forward_sdpa
+
+
+# endregion
+
+# region 画像生成の本体：lpw_stable_diffusion.py （ASL）からコピーして修正
+# https://github.com/huggingface/diffusers/blob/main/examples/community/lpw_stable_diffusion.py
+# Pipelineだけ独立して使えないのと機能追加するのとでコピーして修正
+
+
+class PipelineLike:
+    def __init__(
+        self,
+        device,
+        vae: AutoencoderKL,
+        text_encoders: List[CLIPTextModel],
+        tokenizers: List[CLIPTokenizer],
+        unet: SdxlUNet2DConditionModel,
+        scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler],
+        clip_skip: int,
+    ):
+        super().__init__()
+        self.device = device
+        self.clip_skip = clip_skip
+
+        if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
+            deprecation_message = (
+                f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
+                f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
+                "to update the config accordingly as leaving `steps_offset` might led to incorrect results"
+                " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"
+                " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"
+                " file"
+            )
+            deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)
+            new_config = dict(scheduler.config)
+            new_config["steps_offset"] = 1
+            scheduler._internal_dict = FrozenDict(new_config)
+
+        if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:
+            deprecation_message = (
+                f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."
+                " `clip_sample` should be set to False in the configuration file. Please make sure to update the"
+                " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"
+                " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"
+                " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"
+            )
+            deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)
+            new_config = dict(scheduler.config)
+            new_config["clip_sample"] = False
+            scheduler._internal_dict = FrozenDict(new_config)
+
+        self.vae = vae
+        self.text_encoders = text_encoders
+        self.tokenizers = tokenizers
+        self.unet: SdxlUNet2DConditionModel = unet
+        self.scheduler = scheduler
+        self.safety_checker = None
+
+        # Textual Inversion
+        self.token_replacements_list = []
+        for _ in range(len(self.text_encoders)):
+            self.token_replacements_list.append({})
+
+        # ControlNet # not supported yet
+        self.control_nets: List[ControlNetInfo] = []
+        self.control_net_enabled = True  # control_netsが空ならTrueでもFalseでもControlNetは動作しない
+
+    # Textual Inversion
+    def add_token_replacement(self, text_encoder_index, target_token_id, rep_token_ids):
+        self.token_replacements_list[text_encoder_index][target_token_id] = rep_token_ids
+
+    def set_enable_control_net(self, en: bool):
+        self.control_net_enabled = en
+
+    def get_token_replacer(self, tokenizer):
+        tokenizer_index = self.tokenizers.index(tokenizer)
+        token_replacements = self.token_replacements_list[tokenizer_index]
+
+        def replace_tokens(tokens):
+            # print("replace_tokens", tokens, "=>", token_replacements)
+            if isinstance(tokens, torch.Tensor):
+                tokens = tokens.tolist()
+
+            new_tokens = []
+            for token in tokens:
+                if token in token_replacements:
+                    replacement = token_replacements[token]
+                    new_tokens.extend(replacement)
+                else:
+                    new_tokens.append(token)
+            return new_tokens
+
+        return replace_tokens
+
+    def set_control_nets(self, ctrl_nets):
+        self.control_nets = ctrl_nets
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        prompt: Union[str, List[str]],
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        init_image: Union[torch.FloatTensor, PIL.Image.Image, List[PIL.Image.Image]] = None,
+        mask_image: Union[torch.FloatTensor, PIL.Image.Image, List[PIL.Image.Image]] = None,
+        height: int = 1024,
+        width: int = 1024,
+        original_height: int = None,
+        original_width: int = None,
+        crop_top: int = 0,
+        crop_left: int = 0,
+        num_inference_steps: int = 50,
+        guidance_scale: float = 7.5,
+        negative_scale: float = None,
+        strength: float = 0.8,
+        # num_images_per_prompt: Optional[int] = 1,
+        eta: float = 0.0,
+        generator: Optional[torch.Generator] = None,
+        latents: Optional[torch.FloatTensor] = None,
+        max_embeddings_multiples: Optional[int] = 3,
+        output_type: Optional[str] = "pil",
+        vae_batch_size: float = None,
+        return_latents: bool = False,
+        # return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        is_cancelled_callback: Optional[Callable[[], bool]] = None,
+        callback_steps: Optional[int] = 1,
+        img2img_noise=None,
+        **kwargs,
+    ):
+        # TODO support secondary prompt
+        num_images_per_prompt = 1  # fixed because already prompt is repeated
+
+        if isinstance(prompt, str):
+            batch_size = 1
+            prompt = [prompt]
+        elif isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
+        reginonal_network = " AND " in prompt[0]
+
+        vae_batch_size = (
+            batch_size
+            if vae_batch_size is None
+            else (int(vae_batch_size) if vae_batch_size >= 1 else max(1, int(batch_size * vae_batch_size)))
+        )
+
+        if strength < 0 or strength > 1:
+            raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}")
+
+        if height % 8 != 0 or width % 8 != 0:
+            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
+
+        if (callback_steps is None) or (
+            callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
+        ):
+            raise ValueError(
+                f"`callback_steps` has to be a positive integer but is {callback_steps} of type" f" {type(callback_steps)}."
+            )
+
+        # get prompt text embeddings
+
+        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
+        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
+        # corresponds to doing no classifier free guidance.
+        do_classifier_free_guidance = guidance_scale > 1.0
+
+        if not do_classifier_free_guidance and negative_scale is not None:
+            print(f"negative_scale is ignored if guidance scalle <= 1.0")
+            negative_scale = None
+
+        # get unconditional embeddings for classifier free guidance
+        if negative_prompt is None:
+            negative_prompt = [""] * batch_size
+        elif isinstance(negative_prompt, str):
+            negative_prompt = [negative_prompt] * batch_size
+        if batch_size != len(negative_prompt):
+            raise ValueError(
+                f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
+                f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
+                " the batch size of `prompt`."
+            )
+
+        tes_text_embs = []
+        tes_uncond_embs = []
+        tes_real_uncond_embs = []
+        # use last pool
+        for tokenizer, text_encoder in zip(self.tokenizers, self.text_encoders):
+            token_replacer = self.get_token_replacer(tokenizer)
+
+            text_embeddings, text_pool, uncond_embeddings, uncond_pool, _ = get_weighted_text_embeddings(
+                tokenizer,
+                text_encoder,
+                prompt=prompt,
+                uncond_prompt=negative_prompt if do_classifier_free_guidance else None,
+                max_embeddings_multiples=max_embeddings_multiples,
+                clip_skip=self.clip_skip,
+                token_replacer=token_replacer,
+                device=self.device,
+                **kwargs,
+            )
+            tes_text_embs.append(text_embeddings)
+            tes_uncond_embs.append(uncond_embeddings)
+
+            if negative_scale is not None:
+                _, real_uncond_embeddings, _ = get_weighted_text_embeddings(
+                    token_replacer,
+                    prompt=prompt,  # こちらのトークン長に合わせてuncondを作るので75トークン超で必須
+                    uncond_prompt=[""] * batch_size,
+                    max_embeddings_multiples=max_embeddings_multiples,
+                    clip_skip=self.clip_skip,
+                    token_replacer=token_replacer,
+                    device=self.device,
+                    **kwargs,
+                )
+                tes_real_uncond_embs.append(real_uncond_embeddings)
+
+        # concat text encoder outputs
+        text_embeddings = tes_text_embs[0]
+        uncond_embeddings = tes_uncond_embs[0]
+        for i in range(1, len(tes_text_embs)):
+            text_embeddings = torch.cat([text_embeddings, tes_text_embs[i]], dim=2)  # n,77,2048
+            uncond_embeddings = torch.cat([uncond_embeddings, tes_uncond_embs[i]], dim=2)  # n,77,2048
+
+        if do_classifier_free_guidance:
+            if negative_scale is None:
+                text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
+            else:
+                text_embeddings = torch.cat([uncond_embeddings, text_embeddings, real_uncond_embeddings])
+
+        if self.control_nets:
+            if isinstance(clip_guide_images, PIL.Image.Image):
+                clip_guide_images = [clip_guide_images]
+
+                # ControlNetのhintにguide imageを流用する
+                # 前処理はControlNet側で行う
+
+        # create size embs
+        if original_height is None:
+            original_height = height
+        if original_width is None:
+            original_width = width
+        if crop_top is None:
+            crop_top = 0
+        if crop_left is None:
+            crop_left = 0
+        emb1 = sdxl_train_util.get_timestep_embedding(torch.FloatTensor([original_height, original_width]).unsqueeze(0), 256)
+        emb2 = sdxl_train_util.get_timestep_embedding(torch.FloatTensor([crop_top, crop_left]).unsqueeze(0), 256)
+        emb3 = sdxl_train_util.get_timestep_embedding(torch.FloatTensor([height, width]).unsqueeze(0), 256)
+        c_vector = torch.cat([emb1, emb2, emb3], dim=1).to(self.device, dtype=text_embeddings.dtype)
+        uc_vector = c_vector.clone().to(self.device, dtype=text_embeddings.dtype)
+
+        c_vector = torch.cat([text_pool, c_vector], dim=1)
+        uc_vector = torch.cat([uncond_pool, uc_vector], dim=1)
+
+        vector_embeddings = torch.cat([uc_vector, c_vector])
+
+        # set timesteps
+        self.scheduler.set_timesteps(num_inference_steps, self.device)
+
+        latents_dtype = text_embeddings.dtype
+        init_latents_orig = None
+        mask = None
+
+        if init_image is None:
+            # get the initial random noise unless the user supplied it
+
+            # Unlike in other pipelines, latents need to be generated in the target device
+            # for 1-to-1 results reproducibility with the CompVis implementation.
+            # However this currently doesn't work in `mps`.
+            latents_shape = (
+                batch_size * num_images_per_prompt,
+                self.unet.in_channels,
+                height // 8,
+                width // 8,
+            )
+
+            if latents is None:
+                if self.device.type == "mps":
+                    # randn does not exist on mps
+                    latents = torch.randn(
+                        latents_shape,
+                        generator=generator,
+                        device="cpu",
+                        dtype=latents_dtype,
+                    ).to(self.device)
+                else:
+                    latents = torch.randn(
+                        latents_shape,
+                        generator=generator,
+                        device=self.device,
+                        dtype=latents_dtype,
+                    )
+            else:
+                if latents.shape != latents_shape:
+                    raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}")
+                latents = latents.to(self.device)
+
+            timesteps = self.scheduler.timesteps.to(self.device)
+
+            # scale the initial noise by the standard deviation required by the scheduler
+            latents = latents * self.scheduler.init_noise_sigma
+        else:
+            # image to tensor
+            if isinstance(init_image, PIL.Image.Image):
+                init_image = [init_image]
+            if isinstance(init_image[0], PIL.Image.Image):
+                init_image = [preprocess_image(im) for im in init_image]
+                init_image = torch.cat(init_image)
+            if isinstance(init_image, list):
+                init_image = torch.stack(init_image)
+
+            # mask image to tensor
+            if mask_image is not None:
+                if isinstance(mask_image, PIL.Image.Image):
+                    mask_image = [mask_image]
+                if isinstance(mask_image[0], PIL.Image.Image):
+                    mask_image = torch.cat([preprocess_mask(im) for im in mask_image])  # H*W, 0 for repaint
+
+            # encode the init image into latents and scale the latents
+            init_image = init_image.to(device=self.device, dtype=latents_dtype)
+            if init_image.size()[-2:] == (height // 8, width // 8):
+                init_latents = init_image
+            else:
+                if vae_batch_size >= batch_size:
+                    init_latent_dist = self.vae.encode(init_image.to(self.vae.dtype)).latent_dist
+                    init_latents = init_latent_dist.sample(generator=generator)
+                else:
+                    if torch.cuda.is_available():
+                        torch.cuda.empty_cache()
+                    init_latents = []
+                    for i in tqdm(range(0, min(batch_size, len(init_image)), vae_batch_size)):
+                        init_latent_dist = self.vae.encode(
+                            (init_image[i : i + vae_batch_size] if vae_batch_size > 1 else init_image[i].unsqueeze(0)).to(
+                                self.vae.dtype
+                            )
+                        ).latent_dist
+                        init_latents.append(init_latent_dist.sample(generator=generator))
+                    init_latents = torch.cat(init_latents)
+
+                init_latents = sdxl_model_util.VAE_SCALE_FACTOR * init_latents
+
+            if len(init_latents) == 1:
+                init_latents = init_latents.repeat((batch_size, 1, 1, 1))
+            init_latents_orig = init_latents
+
+            # preprocess mask
+            if mask_image is not None:
+                mask = mask_image.to(device=self.device, dtype=latents_dtype)
+                if len(mask) == 1:
+                    mask = mask.repeat((batch_size, 1, 1, 1))
+
+                # check sizes
+                if not mask.shape == init_latents.shape:
+                    raise ValueError("The mask and init_image should be the same size!")
+
+            # get the original timestep using init_timestep
+            offset = self.scheduler.config.get("steps_offset", 0)
+            init_timestep = int(num_inference_steps * strength) + offset
+            init_timestep = min(init_timestep, num_inference_steps)
+
+            timesteps = self.scheduler.timesteps[-init_timestep]
+            timesteps = torch.tensor([timesteps] * batch_size * num_images_per_prompt, device=self.device)
+
+            # add noise to latents using the timesteps
+            latents = self.scheduler.add_noise(init_latents, img2img_noise, timesteps)
+
+            t_start = max(num_inference_steps - init_timestep + offset, 0)
+            timesteps = self.scheduler.timesteps[t_start:].to(self.device)
+
+        # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
+        # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
+        # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
+        # and should be between [0, 1]
+        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
+        extra_step_kwargs = {}
+        if accepts_eta:
+            extra_step_kwargs["eta"] = eta
+
+        num_latent_input = (3 if negative_scale is not None else 2) if do_classifier_free_guidance else 1
+
+        if self.control_nets:
+            guided_hints = original_control_net.get_guided_hints(self.control_nets, num_latent_input, batch_size, clip_guide_images)
+
+        for i, t in enumerate(tqdm(timesteps)):
+            # expand the latents if we are doing classifier free guidance
+            latent_model_input = latents.repeat((num_latent_input, 1, 1, 1))
+            latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+            # predict the noise residual
+            if self.control_nets and self.control_net_enabled:
+                if reginonal_network:
+                    num_sub_and_neg_prompts = len(text_embeddings) // batch_size
+                    text_emb_last = text_embeddings[num_sub_and_neg_prompts - 2 :: num_sub_and_neg_prompts]  # last subprompt
+                else:
+                    text_emb_last = text_embeddings
+
+                # not working yet
+                noise_pred = original_control_net.call_unet_and_control_net(
+                    i,
+                    num_latent_input,
+                    self.unet,
+                    self.control_nets,
+                    guided_hints,
+                    i / len(timesteps),
+                    latent_model_input,
+                    t,
+                    text_emb_last,
+                ).sample
+            else:
+                noise_pred = self.unet(latent_model_input, t, text_embeddings, vector_embeddings)
+
+            # perform guidance
+            if do_classifier_free_guidance:
+                if negative_scale is None:
+                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(num_latent_input)  # uncond by negative prompt
+                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+                else:
+                    noise_pred_negative, noise_pred_text, noise_pred_uncond = noise_pred.chunk(
+                        num_latent_input
+                    )  # uncond is real uncond
+                    noise_pred = (
+                        noise_pred_uncond
+                        + guidance_scale * (noise_pred_text - noise_pred_uncond)
+                        - negative_scale * (noise_pred_negative - noise_pred_uncond)
+                    )
+
+            # compute the previous noisy sample x_t -> x_t-1
+            latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
+
+            if mask is not None:
+                # masking
+                init_latents_proper = self.scheduler.add_noise(init_latents_orig, img2img_noise, torch.tensor([t]))
+                latents = (init_latents_proper * mask) + (latents * (1 - mask))
+
+            # call the callback, if provided
+            if i % callback_steps == 0:
+                if callback is not None:
+                    callback(i, t, latents)
+                if is_cancelled_callback is not None and is_cancelled_callback():
+                    return None
+
+        if return_latents:
+            return (latents, False)
+
+        latents = 1 / sdxl_model_util.VAE_SCALE_FACTOR * latents
+        if vae_batch_size >= batch_size:
+            image = self.vae.decode(latents.to(self.vae.dtype)).sample
+        else:
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+            images = []
+            for i in tqdm(range(0, batch_size, vae_batch_size)):
+                images.append(
+                    self.vae.decode(
+                        (latents[i : i + vae_batch_size] if vae_batch_size > 1 else latents[i].unsqueeze(0)).to(self.vae.dtype)
+                    ).sample
+                )
+            image = torch.cat(images)
+
+        image = (image / 2 + 0.5).clamp(0, 1)
+
+        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16
+        image = image.cpu().permute(0, 2, 3, 1).float().numpy()
+
+        if output_type == "pil":
+            # image = self.numpy_to_pil(image)
+            image = (image * 255).round().astype("uint8")
+            image = [Image.fromarray(im) for im in image]
+
+        return image
+
+        # return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)
+
+
+re_attention = re.compile(
+    r"""
+\\\(|
+\\\)|
+\\\[|
+\\]|
+\\\\|
+\\|
+\(|
+\[|
+:([+-]?[.\d]+)\)|
+\)|
+]|
+[^\\()\[\]:]+|
+:
+""",
+    re.X,
+)
+
+
+def parse_prompt_attention(text):
+    """
+    Parses a string with attention tokens and returns a list of pairs: text and its associated weight.
+    Accepted tokens are:
+      (abc) - increases attention to abc by a multiplier of 1.1
+      (abc:3.12) - increases attention to abc by a multiplier of 3.12
+      [abc] - decreases attention to abc by a multiplier of 1.1
+      \( - literal character '('
+      \[ - literal character '['
+      \) - literal character ')'
+      \] - literal character ']'
+      \\ - literal character '\'
+      anything else - just text
+    >>> parse_prompt_attention('normal text')
+    [['normal text', 1.0]]
+    >>> parse_prompt_attention('an (important) word')
+    [['an ', 1.0], ['important', 1.1], [' word', 1.0]]
+    >>> parse_prompt_attention('(unbalanced')
+    [['unbalanced', 1.1]]
+    >>> parse_prompt_attention('\(literal\]')
+    [['(literal]', 1.0]]
+    >>> parse_prompt_attention('(unnecessary)(parens)')
+    [['unnecessaryparens', 1.1]]
+    >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')
+    [['a ', 1.0],
+     ['house', 1.5730000000000004],
+     [' ', 1.1],
+     ['on', 1.0],
+     [' a ', 1.1],
+     ['hill', 0.55],
+     [', sun, ', 1.1],
+     ['sky', 1.4641000000000006],
+     ['.', 1.1]]
+    """
+
+    res = []
+    round_brackets = []
+    square_brackets = []
+
+    round_bracket_multiplier = 1.1
+    square_bracket_multiplier = 1 / 1.1
+
+    def multiply_range(start_position, multiplier):
+        for p in range(start_position, len(res)):
+            res[p][1] *= multiplier
+
+    # keep break as separate token
+    text = text.replace("BREAK", "\\BREAK\\")
+
+    for m in re_attention.finditer(text):
+        text = m.group(0)
+        weight = m.group(1)
+
+        if text.startswith("\\"):
+            res.append([text[1:], 1.0])
+        elif text == "(":
+            round_brackets.append(len(res))
+        elif text == "[":
+            square_brackets.append(len(res))
+        elif weight is not None and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), float(weight))
+        elif text == ")" and len(round_brackets) > 0:
+            multiply_range(round_brackets.pop(), round_bracket_multiplier)
+        elif text == "]" and len(square_brackets) > 0:
+            multiply_range(square_brackets.pop(), square_bracket_multiplier)
+        else:
+            res.append([text, 1.0])
+
+    for pos in round_brackets:
+        multiply_range(pos, round_bracket_multiplier)
+
+    for pos in square_brackets:
+        multiply_range(pos, square_bracket_multiplier)
+
+    if len(res) == 0:
+        res = [["", 1.0]]
+
+    # merge runs of identical weights
+    i = 0
+    while i + 1 < len(res):
+        if res[i][1] == res[i + 1][1] and res[i][0].strip() != "BREAK" and res[i + 1][0].strip() != "BREAK":
+            res[i][0] += res[i + 1][0]
+            res.pop(i + 1)
+        else:
+            i += 1
+
+    return res
+
+
+def get_prompts_with_weights(tokenizer: CLIPTokenizer, token_replacer, prompt: List[str], max_length: int):
+    r"""
+    Tokenize a list of prompts and return its tokens with weights of each token.
+    No padding, starting or ending token is included.
+    """
+    tokens = []
+    weights = []
+    truncated = False
+
+    for text in prompt:
+        texts_and_weights = parse_prompt_attention(text)
+        text_token = []
+        text_weight = []
+        for word, weight in texts_and_weights:
+            if word.strip() == "BREAK":
+                # pad until next multiple of tokenizer's max token length
+                pad_len = tokenizer.model_max_length - (len(text_token) % tokenizer.model_max_length)
+                print(f"BREAK pad_len: {pad_len}")
+                for i in range(pad_len):
+                    # v2のときEOSをつけるべきかどうかわからないぜ
+                    # if i == 0:
+                    #     text_token.append(tokenizer.eos_token_id)
+                    # else:
+                    text_token.append(tokenizer.pad_token_id)
+                    text_weight.append(1.0)
+                continue
+
+            # tokenize and discard the starting and the ending token
+            token = tokenizer(word).input_ids[1:-1]
+
+            token = token_replacer(token)  # for Textual Inversion
+
+            text_token += token
+            # copy the weight by length of token
+            text_weight += [weight] * len(token)
+            # stop if the text is too long (longer than truncation limit)
+            if len(text_token) > max_length:
+                truncated = True
+                break
+        # truncate
+        if len(text_token) > max_length:
+            truncated = True
+            text_token = text_token[:max_length]
+            text_weight = text_weight[:max_length]
+        tokens.append(text_token)
+        weights.append(text_weight)
+    if truncated:
+        print("warning: Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples")
+    return tokens, weights
+
+
+def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, pad, no_boseos_middle=True, chunk_length=77):
+    r"""
+    Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length.
+    """
+    max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)
+    weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length
+    for i in range(len(tokens)):
+        tokens[i] = [bos] + tokens[i] + [eos] + [pad] * (max_length - 2 - len(tokens[i]))
+        if no_boseos_middle:
+            weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))
+        else:
+            w = []
+            if len(weights[i]) == 0:
+                w = [1.0] * weights_length
+            else:
+                for j in range(max_embeddings_multiples):
+                    w.append(1.0)  # weight for starting token in this chunk
+                    w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]
+                    w.append(1.0)  # weight for ending token in this chunk
+                w += [1.0] * (weights_length - len(w))
+            weights[i] = w[:]
+
+    return tokens, weights
+
+
+def get_unweighted_text_embeddings(
+    text_encoder: CLIPTextModel,
+    text_input: torch.Tensor,
+    chunk_length: int,
+    clip_skip: int,
+    eos: int,
+    pad: int,
+    no_boseos_middle: Optional[bool] = True,
+):
+    """
+    When the length of tokens is a multiple of the capacity of the text encoder,
+    it should be split into chunks and sent to the text encoder individually.
+    """
+    max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)
+    if max_embeddings_multiples > 1:
+        text_embeddings = []
+        pool = None
+        for i in range(max_embeddings_multiples):
+            # extract the i-th chunk
+            text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()
+
+            # cover the head and the tail by the starting and the ending tokens
+            text_input_chunk[:, 0] = text_input[0, 0]
+            if pad == eos:  # v1
+                text_input_chunk[:, -1] = text_input[0, -1]
+            else:  # v2
+                for j in range(len(text_input_chunk)):
+                    if text_input_chunk[j, -1] != eos and text_input_chunk[j, -1] != pad:  # 最後に普通の文字がある
+                        text_input_chunk[j, -1] = eos
+                    if text_input_chunk[j, 1] == pad:  # BOSだけであとはPAD
+                        text_input_chunk[j, 1] = eos
+
+            # -2 is same for Text Encoder 1 and 2
+            enc_out = text_encoder(text_input_chunk, output_hidden_states=True, return_dict=True)
+            text_embedding = enc_out["hidden_states"][-2]
+            if pool is None:
+                pool = enc_out["text_embeds"]  # use 1st chunk
+
+            if no_boseos_middle:
+                if i == 0:
+                    # discard the ending token
+                    text_embedding = text_embedding[:, :-1]
+                elif i == max_embeddings_multiples - 1:
+                    # discard the starting token
+                    text_embedding = text_embedding[:, 1:]
+                else:
+                    # discard both starting and ending tokens
+                    text_embedding = text_embedding[:, 1:-1]
+
+            text_embeddings.append(text_embedding)
+        text_embeddings = torch.concat(text_embeddings, axis=1)
+    else:
+        enc_out = text_encoder(text_input, output_hidden_states=True, return_dict=True)
+        text_embeddings = enc_out["hidden_states"][-2]
+        pool = enc_out.get("text_embeds", None)  # text encoder 1 doesn't return this
+    return text_embeddings, pool
+
+
+def get_weighted_text_embeddings(
+    tokenizer: CLIPTokenizer,
+    text_encoder: CLIPTextModel,
+    prompt: Union[str, List[str]],
+    uncond_prompt: Optional[Union[str, List[str]]] = None,
+    max_embeddings_multiples: Optional[int] = 1,
+    no_boseos_middle: Optional[bool] = False,
+    skip_parsing: Optional[bool] = False,
+    skip_weighting: Optional[bool] = False,
+    clip_skip=None,
+    token_replacer=None,
+    device=None,
+    **kwargs,
+):
+    max_length = (tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+    if isinstance(prompt, str):
+        prompt = [prompt]
+
+    # split the prompts with "AND". each prompt must have the same number of splits
+    new_prompts = []
+    for p in prompt:
+        new_prompts.extend(p.split(" AND "))
+    prompt = new_prompts
+
+    if not skip_parsing:
+        prompt_tokens, prompt_weights = get_prompts_with_weights(tokenizer, token_replacer, prompt, max_length - 2)
+        if uncond_prompt is not None:
+            if isinstance(uncond_prompt, str):
+                uncond_prompt = [uncond_prompt]
+            uncond_tokens, uncond_weights = get_prompts_with_weights(tokenizer, token_replacer, uncond_prompt, max_length - 2)
+    else:
+        prompt_tokens = [token[1:-1] for token in tokenizer(prompt, max_length=max_length, truncation=True).input_ids]
+        prompt_weights = [[1.0] * len(token) for token in prompt_tokens]
+        if uncond_prompt is not None:
+            if isinstance(uncond_prompt, str):
+                uncond_prompt = [uncond_prompt]
+            uncond_tokens = [token[1:-1] for token in tokenizer(uncond_prompt, max_length=max_length, truncation=True).input_ids]
+            uncond_weights = [[1.0] * len(token) for token in uncond_tokens]
+
+    # round up the longest length of tokens to a multiple of (model_max_length - 2)
+    max_length = max([len(token) for token in prompt_tokens])
+    if uncond_prompt is not None:
+        max_length = max(max_length, max([len(token) for token in uncond_tokens]))
+
+    max_embeddings_multiples = min(
+        max_embeddings_multiples,
+        (max_length - 1) // (tokenizer.model_max_length - 2) + 1,
+    )
+    max_embeddings_multiples = max(1, max_embeddings_multiples)
+    max_length = (tokenizer.model_max_length - 2) * max_embeddings_multiples + 2
+
+    # pad the length of tokens and weights
+    bos = tokenizer.bos_token_id
+    eos = tokenizer.eos_token_id
+    pad = tokenizer.pad_token_id
+    prompt_tokens, prompt_weights = pad_tokens_and_weights(
+        prompt_tokens,
+        prompt_weights,
+        max_length,
+        bos,
+        eos,
+        pad,
+        no_boseos_middle=no_boseos_middle,
+        chunk_length=tokenizer.model_max_length,
+    )
+    prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device=device)
+    if uncond_prompt is not None:
+        uncond_tokens, uncond_weights = pad_tokens_and_weights(
+            uncond_tokens,
+            uncond_weights,
+            max_length,
+            bos,
+            eos,
+            pad,
+            no_boseos_middle=no_boseos_middle,
+            chunk_length=tokenizer.model_max_length,
+        )
+        uncond_tokens = torch.tensor(uncond_tokens, dtype=torch.long, device=device)
+
+    # get the embeddings
+    text_embeddings, text_pool = get_unweighted_text_embeddings(
+        text_encoder,
+        prompt_tokens,
+        tokenizer.model_max_length,
+        clip_skip,
+        eos,
+        pad,
+        no_boseos_middle=no_boseos_middle,
+    )
+    prompt_weights = torch.tensor(prompt_weights, dtype=text_embeddings.dtype, device=device)
+    if uncond_prompt is not None:
+        uncond_embeddings, uncond_pool = get_unweighted_text_embeddings(
+            text_encoder,
+            uncond_tokens,
+            tokenizer.model_max_length,
+            clip_skip,
+            eos,
+            pad,
+            no_boseos_middle=no_boseos_middle,
+        )
+        uncond_weights = torch.tensor(uncond_weights, dtype=uncond_embeddings.dtype, device=device)
+
+    # assign weights to the prompts and normalize in the sense of mean
+    # TODO: should we normalize by chunk or in a whole (current implementation)?
+    # →全体でいいんじゃないかな
+    if (not skip_parsing) and (not skip_weighting):
+        previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+        text_embeddings *= prompt_weights.unsqueeze(-1)
+        current_mean = text_embeddings.float().mean(axis=[-2, -1]).to(text_embeddings.dtype)
+        text_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)
+        if uncond_prompt is not None:
+            previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)
+            uncond_embeddings *= uncond_weights.unsqueeze(-1)
+            current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to(uncond_embeddings.dtype)
+            uncond_embeddings *= (previous_mean / current_mean).unsqueeze(-1).unsqueeze(-1)
+
+    if uncond_prompt is not None:
+        return text_embeddings, text_pool, uncond_embeddings, uncond_pool, prompt_tokens
+    return text_embeddings, text_pool, None, None, prompt_tokens
+
+
+def preprocess_image(image):
+    w, h = image.size
+    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32
+    image = image.resize((w, h), resample=PIL.Image.LANCZOS)
+    image = np.array(image).astype(np.float32) / 255.0
+    image = image[None].transpose(0, 3, 1, 2)
+    image = torch.from_numpy(image)
+    return 2.0 * image - 1.0
+
+
+def preprocess_mask(mask):
+    mask = mask.convert("L")
+    w, h = mask.size
+    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32
+    mask = mask.resize((w // 8, h // 8), resample=PIL.Image.BILINEAR)  # LANCZOS)
+    mask = np.array(mask).astype(np.float32) / 255.0
+    mask = np.tile(mask, (4, 1, 1))
+    mask = mask[None].transpose(0, 1, 2, 3)  # what does this step do?
+    mask = 1 - mask  # repaint white, keep black
+    mask = torch.from_numpy(mask)
+    return mask
+
+
+# regular expression for dynamic prompt:
+# starts and ends with "{" and "}"
+# contains at least one variant divided by "|"
+# optional framgments divided by "$$" at start
+# if the first fragment is "E" or "e", enumerate all variants
+# if the second fragment is a number or two numbers, repeat the variants in the range
+# if the third fragment is a string, use it as a separator
+
+RE_DYNAMIC_PROMPT = re.compile(r"\{((e|E)\$\$)?(([\d\-]+)\$\$)?(([^\|\}]+?)\$\$)?(.+?((\|).+?)*?)\}")
+
+
+def handle_dynamic_prompt_variants(prompt, repeat_count):
+    founds = list(RE_DYNAMIC_PROMPT.finditer(prompt))
+    if not founds:
+        return [prompt]
+
+    # make each replacement for each variant
+    enumerating = False
+    replacers = []
+    for found in founds:
+        # if "e$$" is found, enumerate all variants
+        found_enumerating = found.group(2) is not None
+        enumerating = enumerating or found_enumerating
+
+        separator = ", " if found.group(6) is None else found.group(6)
+        variants = found.group(7).split("|")
+
+        # parse count range
+        count_range = found.group(4)
+        if count_range is None:
+            count_range = [1, 1]
+        else:
+            count_range = count_range.split("-")
+            if len(count_range) == 1:
+                count_range = [int(count_range[0]), int(count_range[0])]
+            elif len(count_range) == 2:
+                count_range = [int(count_range[0]), int(count_range[1])]
+            else:
+                print(f"invalid count range: {count_range}")
+                count_range = [1, 1]
+            if count_range[0] > count_range[1]:
+                count_range = [count_range[1], count_range[0]]
+            if count_range[0] < 0:
+                count_range[0] = 0
+            if count_range[1] > len(variants):
+                count_range[1] = len(variants)
+
+        if found_enumerating:
+            # make function to enumerate all combinations
+            def make_replacer_enum(vari, cr, sep):
+                def replacer():
+                    values = []
+                    for count in range(cr[0], cr[1] + 1):
+                        for comb in itertools.combinations(vari, count):
+                            values.append(sep.join(comb))
+                    return values
+
+                return replacer
+
+            replacers.append(make_replacer_enum(variants, count_range, separator))
+        else:
+            # make function to choose random combinations
+            def make_replacer_single(vari, cr, sep):
+                def replacer():
+                    count = random.randint(cr[0], cr[1])
+                    comb = random.sample(vari, count)
+                    return [sep.join(comb)]
+
+                return replacer
+
+            replacers.append(make_replacer_single(variants, count_range, separator))
+
+    # make each prompt
+    if not enumerating:
+        # if not enumerating, repeat the prompt, replace each variant randomly
+        prompts = []
+        for _ in range(repeat_count):
+            current = prompt
+            for found, replacer in zip(founds, replacers):
+                current = current.replace(found.group(0), replacer()[0], 1)
+            prompts.append(current)
+    else:
+        # if enumerating, iterate all combinations for previous prompts
+        prompts = [prompt]
+
+        for found, replacer in zip(founds, replacers):
+            if found.group(2) is not None:
+                # make all combinations for existing prompts
+                new_prompts = []
+                for current in prompts:
+                    replecements = replacer()
+                    for replecement in replecements:
+                        new_prompts.append(current.replace(found.group(0), replecement, 1))
+                prompts = new_prompts
+
+        for found, replacer in zip(founds, replacers):
+            # make random selection for existing prompts
+            if found.group(2) is None:
+                for i in range(len(prompts)):
+                    prompts[i] = prompts[i].replace(found.group(0), replacer()[0], 1)
+
+    return prompts
+
+
+# endregion
+
+
+# def load_clip_l14_336(dtype):
+#   print(f"loading CLIP: {CLIP_ID_L14_336}")
+#   text_encoder = CLIPTextModel.from_pretrained(CLIP_ID_L14_336, torch_dtype=dtype)
+#   return text_encoder
+
+
+class BatchDataBase(NamedTuple):
+    # バッチ分割が必要ないデータ
+    step: int
+    prompt: str
+    negative_prompt: str
+    seed: int
+    init_image: Any
+    mask_image: Any
+    clip_prompt: str
+    guide_image: Any
+
+
+class BatchDataExt(NamedTuple):
+    # バッチ分割が必要なデータ
+    width: int
+    height: int
+    original_width: int
+    original_height: int
+    crop_left: int
+    crop_top: int
+    steps: int
+    scale: float
+    negative_scale: float
+    strength: float
+    network_muls: Tuple[float]
+    num_sub_prompts: int
+
+
+class BatchData(NamedTuple):
+    return_latents: bool
+    base: BatchDataBase
+    ext: BatchDataExt
+
+
+def main(args):
+    if args.fp16:
+        dtype = torch.float16
+    elif args.bf16:
+        dtype = torch.bfloat16
+    else:
+        dtype = torch.float32
+
+    highres_fix = args.highres_fix_scale is not None
+    # assert not highres_fix or args.image_path is None, f"highres_fix doesn't work with img2img / highres_fixはimg2imgと同時に使えません"
+
+    # モデルを読み込む
+    if not os.path.isfile(args.ckpt):  # ファイルがないならパターンで探し、一つだけ該当すればそれを使う
+        files = glob.glob(args.ckpt)
+        if len(files) == 1:
+            args.ckpt = files[0]
+
+    use_stable_diffusion_format = os.path.isfile(args.ckpt)
+    assert use_stable_diffusion_format, "Diffusers pretrained models are not supported yet"
+    print("load StableDiffusion checkpoint")
+    text_encoder1, text_encoder2, vae, unet, _, _ = sdxl_model_util.load_models_from_sdxl_checkpoint(
+        sdxl_model_util.MODEL_VERSION_SDXL_BASE_V0_9, args.ckpt, "cpu"
+    )
+    # else:
+    #     print("load Diffusers pretrained models")
+    # TODO use Diffusers 0.18.1 and support SDXL pipeline
+    # raise NotImplementedError("Diffusers pretrained models are not supported yet")
+    # loading_pipe = StableDiffusionXLPipeline.from_pretrained(args.ckpt, safety_checker=None, torch_dtype=dtype)
+    # text_encoder = loading_pipe.text_encoder
+    # vae = loading_pipe.vae
+    # unet = loading_pipe.unet
+    # tokenizer = loading_pipe.tokenizer
+    # del loading_pipe
+
+    # # Diffusers U-Net to original U-Net
+    # original_unet = SdxlUNet2DConditionModel(
+    #     unet.config.sample_size,
+    #     unet.config.attention_head_dim,
+    #     unet.config.cross_attention_dim,
+    #     unet.config.use_linear_projection,
+    #     unet.config.upcast_attention,
+    # )
+    # original_unet.load_state_dict(unet.state_dict())
+    # unet = original_unet
+
+    # VAEを読み込む
+    if args.vae is not None:
+        vae = model_util.load_vae(args.vae, dtype)
+        print("additional VAE loaded")
+
+    # xformers、Hypernetwork対応
+    if not args.diffusers_xformers:
+        mem_eff = not (args.xformers or args.sdpa)
+        replace_unet_modules(unet, mem_eff, args.xformers, args.sdpa)
+        replace_vae_modules(vae, mem_eff, args.xformers, args.sdpa)
+
+    # tokenizerを読み込む
+    print("loading tokenizer")
+    if use_stable_diffusion_format:
+        tokenizer1, tokenizer2 = sdxl_train_util.load_tokenizers(args)
+
+    # schedulerを用意する
+    sched_init_args = {}
+    scheduler_num_noises_per_step = 1
+    if args.sampler == "ddim":
+        scheduler_cls = DDIMScheduler
+        scheduler_module = diffusers.schedulers.scheduling_ddim
+    elif args.sampler == "ddpm":  # ddpmはおかしくなるのでoptionから外してある
+        scheduler_cls = DDPMScheduler
+        scheduler_module = diffusers.schedulers.scheduling_ddpm
+    elif args.sampler == "pndm":
+        scheduler_cls = PNDMScheduler
+        scheduler_module = diffusers.schedulers.scheduling_pndm
+    elif args.sampler == "lms" or args.sampler == "k_lms":
+        scheduler_cls = LMSDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_lms_discrete
+    elif args.sampler == "euler" or args.sampler == "k_euler":
+        scheduler_cls = EulerDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_euler_discrete
+    elif args.sampler == "euler_a" or args.sampler == "k_euler_a":
+        scheduler_cls = EulerAncestralDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_euler_ancestral_discrete
+    elif args.sampler == "dpmsolver" or args.sampler == "dpmsolver++":
+        scheduler_cls = DPMSolverMultistepScheduler
+        sched_init_args["algorithm_type"] = args.sampler
+        scheduler_module = diffusers.schedulers.scheduling_dpmsolver_multistep
+    elif args.sampler == "dpmsingle":
+        scheduler_cls = DPMSolverSinglestepScheduler
+        scheduler_module = diffusers.schedulers.scheduling_dpmsolver_singlestep
+    elif args.sampler == "heun":
+        scheduler_cls = HeunDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_heun_discrete
+    elif args.sampler == "dpm_2" or args.sampler == "k_dpm_2":
+        scheduler_cls = KDPM2DiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_k_dpm_2_discrete
+    elif args.sampler == "dpm_2_a" or args.sampler == "k_dpm_2_a":
+        scheduler_cls = KDPM2AncestralDiscreteScheduler
+        scheduler_module = diffusers.schedulers.scheduling_k_dpm_2_ancestral_discrete
+        scheduler_num_noises_per_step = 2
+
+    # samplerの乱数をあらかじめ指定するための処理
+
+    # replace randn
+    class NoiseManager:
+        def __init__(self):
+            self.sampler_noises = None
+            self.sampler_noise_index = 0
+
+        def reset_sampler_noises(self, noises):
+            self.sampler_noise_index = 0
+            self.sampler_noises = noises
+
+        def randn(self, shape, device=None, dtype=None, layout=None, generator=None):
+            # print("replacing", shape, len(self.sampler_noises), self.sampler_noise_index)
+            if self.sampler_noises is not None and self.sampler_noise_index < len(self.sampler_noises):
+                noise = self.sampler_noises[self.sampler_noise_index]
+                if shape != noise.shape:
+                    noise = None
+            else:
+                noise = None
+
+            if noise == None:
+                print(f"unexpected noise request: {self.sampler_noise_index}, {shape}")
+                noise = torch.randn(shape, dtype=dtype, device=device, generator=generator)
+
+            self.sampler_noise_index += 1
+            return noise
+
+    class TorchRandReplacer:
+        def __init__(self, noise_manager):
+            self.noise_manager = noise_manager
+
+        def __getattr__(self, item):
+            if item == "randn":
+                return self.noise_manager.randn
+            if hasattr(torch, item):
+                return getattr(torch, item)
+            raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, item))
+
+    noise_manager = NoiseManager()
+    if scheduler_module is not None:
+        scheduler_module.torch = TorchRandReplacer(noise_manager)
+
+    scheduler = scheduler_cls(
+        num_train_timesteps=SCHEDULER_TIMESTEPS,
+        beta_start=SCHEDULER_LINEAR_START,
+        beta_end=SCHEDULER_LINEAR_END,
+        beta_schedule=SCHEDLER_SCHEDULE,
+        **sched_init_args,
+    )
+
+    # clip_sample=Trueにする
+    if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is False:
+        print("set clip_sample to True")
+        scheduler.config.clip_sample = True
+
+    # deviceを決定する
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # "mps"を考量してない
+
+    # custom pipelineをコピったやつを生成する
+    if args.vae_slices:
+        from library.slicing_vae import SlicingAutoencoderKL
+
+        sli_vae = SlicingAutoencoderKL(
+            act_fn="silu",
+            block_out_channels=(128, 256, 512, 512),
+            down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D"],
+            in_channels=3,
+            latent_channels=4,
+            layers_per_block=2,
+            norm_num_groups=32,
+            out_channels=3,
+            sample_size=512,
+            up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"],
+            num_slices=args.vae_slices,
+        )
+        sli_vae.load_state_dict(vae.state_dict())  # vaeのパラメータをコピーする
+        vae = sli_vae
+        del sli_vae
+
+    vae_dtype = dtype
+    if args.no_half_vae:
+        print("set vae_dtype to float32")
+        vae_dtype = torch.float32
+    vae.to(vae_dtype).to(device)
+
+    text_encoder1.to(dtype).to(device)
+    text_encoder2.to(dtype).to(device)
+    unet.to(dtype).to(device)
+
+    # networkを組み込む
+    if args.network_module:
+        networks = []
+        network_default_muls = []
+        network_pre_calc = args.network_pre_calc
+
+        for i, network_module in enumerate(args.network_module):
+            print("import network module:", network_module)
+            imported_module = importlib.import_module(network_module)
+
+            network_mul = 1.0 if args.network_mul is None or len(args.network_mul) <= i else args.network_mul[i]
+            network_default_muls.append(network_mul)
+
+            net_kwargs = {}
+            if args.network_args and i < len(args.network_args):
+                network_args = args.network_args[i]
+                # TODO escape special chars
+                network_args = network_args.split(";")
+                for net_arg in network_args:
+                    key, value = net_arg.split("=")
+                    net_kwargs[key] = value
+
+            if args.network_weights and i < len(args.network_weights):
+                network_weight = args.network_weights[i]
+                print("load network weights from:", network_weight)
+
+                if model_util.is_safetensors(network_weight) and args.network_show_meta:
+                    from safetensors.torch import safe_open
+
+                    with safe_open(network_weight, framework="pt") as f:
+                        metadata = f.metadata()
+                    if metadata is not None:
+                        print(f"metadata for: {network_weight}: {metadata}")
+
+                network, weights_sd = imported_module.create_network_from_weights(
+                    network_mul, network_weight, vae, [text_encoder1, text_encoder2], unet, for_inference=True, **net_kwargs
+                )
+            else:
+                raise ValueError("No weight. Weight is required.")
+            if network is None:
+                return
+
+            mergeable = network.is_mergeable()
+            if args.network_merge and not mergeable:
+                print("network is not mergiable. ignore merge option.")
+
+            if not args.network_merge or not mergeable:
+                network.apply_to([text_encoder1, text_encoder2], unet)
+                info = network.load_state_dict(weights_sd, False)  # network.load_weightsを使うようにするとよい
+                print(f"weights are loaded: {info}")
+
+                if args.opt_channels_last:
+                    network.to(memory_format=torch.channels_last)
+                network.to(dtype).to(device)
+
+                if network_pre_calc:
+                    print("backup original weights")
+                    network.backup_weights()
+
+                networks.append(network)
+            else:
+                network.merge_to([text_encoder1, text_encoder2], unet, weights_sd, dtype, device)
+
+    else:
+        networks = []
+
+    # upscalerの指定があれば取得する
+    upscaler = None
+    if args.highres_fix_upscaler:
+        print("import upscaler module:", args.highres_fix_upscaler)
+        imported_module = importlib.import_module(args.highres_fix_upscaler)
+
+        us_kwargs = {}
+        if args.highres_fix_upscaler_args:
+            for net_arg in args.highres_fix_upscaler_args.split(";"):
+                key, value = net_arg.split("=")
+                us_kwargs[key] = value
+
+        print("create upscaler")
+        upscaler = imported_module.create_upscaler(**us_kwargs)
+        upscaler.to(dtype).to(device)
+
+    # ControlNetの処理
+    control_nets: List[ControlNetInfo] = []
+    if args.control_net_models:
+        for i, model in enumerate(args.control_net_models):
+            prep_type = None if not args.control_net_preps or len(args.control_net_preps) <= i else args.control_net_preps[i]
+            weight = 1.0 if not args.control_net_weights or len(args.control_net_weights) <= i else args.control_net_weights[i]
+            ratio = 1.0 if not args.control_net_ratios or len(args.control_net_ratios) <= i else args.control_net_ratios[i]
+
+            ctrl_unet, ctrl_net = original_control_net.load_control_net(False, unet, model)
+            prep = original_control_net.load_preprocess(prep_type)
+            control_nets.append(ControlNetInfo(ctrl_unet, ctrl_net, prep, weight, ratio))
+
+    if args.opt_channels_last:
+        print(f"set optimizing: channels last")
+        text_encoder1.to(memory_format=torch.channels_last)
+        text_encoder2.to(memory_format=torch.channels_last)
+        vae.to(memory_format=torch.channels_last)
+        unet.to(memory_format=torch.channels_last)
+        if networks:
+            for network in networks:
+                network.to(memory_format=torch.channels_last)
+
+        for cn in control_nets:
+            cn.unet.to(memory_format=torch.channels_last)
+            cn.net.to(memory_format=torch.channels_last)
+
+    pipe = PipelineLike(
+        device,
+        vae,
+        [text_encoder1, text_encoder2],
+        [tokenizer1, tokenizer2],
+        unet,
+        scheduler,
+        args.clip_skip,
+    )
+    pipe.set_control_nets(control_nets)
+    print("pipeline is ready.")
+
+    if args.diffusers_xformers:
+        pipe.enable_xformers_memory_efficient_attention()
+
+    #  Textual Inversionを処理する
+    if args.textual_inversion_embeddings:
+        token_ids_embeds1 = []
+        token_ids_embeds2 = []
+        for embeds_file in args.textual_inversion_embeddings:
+            if model_util.is_safetensors(embeds_file):
+                from safetensors.torch import load_file
+
+                data = load_file(embeds_file)
+            else:
+                data = torch.load(embeds_file, map_location="cpu")
+
+            if "string_to_param" in data:
+                data = data["string_to_param"]
+
+            embeds1 = data["clip_l"]  # text encoder 1
+            embeds2 = data["clip_g"]  # text encoder 2
+
+            num_vectors_per_token = embeds1.size()[0]
+            token_string = os.path.splitext(os.path.basename(embeds_file))[0]
+
+            # remove non-alphabet characters to avoid splitting by tokenizer
+            # TODO make random alphabet string
+            token_string = "".join([c for c in token_string if c.isalpha()])  
+            
+            token_strings = [token_string] + [f"{token_string}{chr(ord('a') + i)}" for i in range(num_vectors_per_token - 1)]
+
+            # add new word to tokenizer, count is num_vectors_per_token
+            num_added_tokens1 = tokenizer1.add_tokens(token_strings)
+            num_added_tokens2 = tokenizer2.add_tokens(token_strings)
+            assert num_added_tokens1 == num_vectors_per_token and num_added_tokens2 == num_vectors_per_token, (
+                f"tokenizer has same word to token string (filename). characters except alphabet are removed: {embeds_file}"
+                + f" / 指定した名前（ファイル名）のトークンが既に存在します。アルファベット以外の文字は削除されます: {embeds_file}"
+            )
+
+            token_ids1 = tokenizer1.convert_tokens_to_ids(token_strings)
+            token_ids2 = tokenizer2.convert_tokens_to_ids(token_strings)
+            print(f"Textual Inversion embeddings `{token_string}` loaded. Tokens are added: {token_ids1} and {token_ids2}")
+            assert (
+                min(token_ids1) == token_ids1[0] and token_ids1[-1] == token_ids1[0] + len(token_ids1) - 1
+            ), f"token ids1 is not ordered"
+            assert (
+                min(token_ids2) == token_ids2[0] and token_ids2[-1] == token_ids2[0] + len(token_ids2) - 1
+            ), f"token ids2 is not ordered"
+            assert len(tokenizer1) - 1 == token_ids1[-1], f"token ids 1 is not end of tokenize: {len(tokenizer1)}"
+            assert len(tokenizer2) - 1 == token_ids2[-1], f"token ids 2 is not end of tokenize: {len(tokenizer2)}"
+
+            if num_vectors_per_token > 1:
+                pipe.add_token_replacement(0, token_ids1[0], token_ids1)  # hoge -> hoge, hogea, hogeb, ...
+                pipe.add_token_replacement(1, token_ids2[0], token_ids2)
+
+            token_ids_embeds1.append((token_ids1, embeds1))
+            token_ids_embeds2.append((token_ids2, embeds2))
+
+        text_encoder1.resize_token_embeddings(len(tokenizer1))
+        text_encoder2.resize_token_embeddings(len(tokenizer2))
+        token_embeds1 = text_encoder1.get_input_embeddings().weight.data
+        token_embeds2 = text_encoder2.get_input_embeddings().weight.data
+        for token_ids, embeds in token_ids_embeds1:
+            for token_id, embed in zip(token_ids, embeds):
+                token_embeds1[token_id] = embed
+        for token_ids, embeds in token_ids_embeds2:
+            for token_id, embed in zip(token_ids, embeds):
+                token_embeds2[token_id] = embed
+
+    # promptを取得する
+    if args.from_file is not None:
+        print(f"reading prompts from {args.from_file}")
+        with open(args.from_file, "r", encoding="utf-8") as f:
+            prompt_list = f.read().splitlines()
+            prompt_list = [d for d in prompt_list if len(d.strip()) > 0]
+    elif args.prompt is not None:
+        prompt_list = [args.prompt]
+    else:
+        prompt_list = []
+
+    if args.interactive:
+        args.n_iter = 1
+
+    # img2imgの前処理、画像の読み込みなど
+    def load_images(path):
+        if os.path.isfile(path):
+            paths = [path]
+        else:
+            paths = (
+                glob.glob(os.path.join(path, "*.png"))
+                + glob.glob(os.path.join(path, "*.jpg"))
+                + glob.glob(os.path.join(path, "*.jpeg"))
+                + glob.glob(os.path.join(path, "*.webp"))
+            )
+            paths.sort()
+
+        images = []
+        for p in paths:
+            image = Image.open(p)
+            if image.mode != "RGB":
+                print(f"convert image to RGB from {image.mode}: {p}")
+                image = image.convert("RGB")
+            images.append(image)
+
+        return images
+
+    def resize_images(imgs, size):
+        resized = []
+        for img in imgs:
+            r_img = img.resize(size, Image.Resampling.LANCZOS)
+            if hasattr(img, "filename"):  # filename属性がない場合があるらしい
+                r_img.filename = img.filename
+            resized.append(r_img)
+        return resized
+
+    if args.image_path is not None:
+        print(f"load image for img2img: {args.image_path}")
+        init_images = load_images(args.image_path)
+        assert len(init_images) > 0, f"No image / 画像がありません: {args.image_path}"
+        print(f"loaded {len(init_images)} images for img2img")
+    else:
+        init_images = None
+
+    if args.mask_path is not None:
+        print(f"load mask for inpainting: {args.mask_path}")
+        mask_images = load_images(args.mask_path)
+        assert len(mask_images) > 0, f"No mask image / マスク画像がありません: {args.image_path}"
+        print(f"loaded {len(mask_images)} mask images for inpainting")
+    else:
+        mask_images = None
+
+    # promptがないとき、画像のPngInfoから取得する
+    if init_images is not None and len(prompt_list) == 0 and not args.interactive:
+        print("get prompts from images' meta data")
+        for img in init_images:
+            if "prompt" in img.text:
+                prompt = img.text["prompt"]
+                if "negative-prompt" in img.text:
+                    prompt += " --n " + img.text["negative-prompt"]
+                prompt_list.append(prompt)
+
+        # プロンプトと画像を一致させるため指定回数だけ繰り返す（画像を増幅する）
+        l = []
+        for im in init_images:
+            l.extend([im] * args.images_per_prompt)
+        init_images = l
+
+        if mask_images is not None:
+            l = []
+            for im in mask_images:
+                l.extend([im] * args.images_per_prompt)
+            mask_images = l
+
+    # 画像サイズにオプション指定があるときはリサイズする
+    if args.W is not None and args.H is not None:
+        # highres fix を考慮に入れる
+        w, h = args.W, args.H
+        if highres_fix:
+            w = int(w * args.highres_fix_scale + 0.5)
+            h = int(h * args.highres_fix_scale + 0.5)
+
+        if init_images is not None:
+            print(f"resize img2img source images to {w}*{h}")
+            init_images = resize_images(init_images, (w, h))
+        if mask_images is not None:
+            print(f"resize img2img mask images to {w}*{h}")
+            mask_images = resize_images(mask_images, (w, h))
+
+    regional_network = False
+    if networks and mask_images:
+        # mask を領域情報として流用する、現在は一回のコマンド呼び出しで1枚だけ対応
+        regional_network = True
+        print("use mask as region")
+
+        size = None
+        for i, network in enumerate(networks):
+            if i < 3:
+                np_mask = np.array(mask_images[0])
+                np_mask = np_mask[:, :, i]
+                size = np_mask.shape
+            else:
+                np_mask = np.full(size, 255, dtype=np.uint8)
+            mask = torch.from_numpy(np_mask.astype(np.float32) / 255.0)
+            network.set_region(i, i == len(networks) - 1, mask)
+        mask_images = None
+
+    prev_image = None  # for VGG16 guided
+    if args.guide_image_path is not None:
+        print(f"load image for ControlNet guidance: {args.guide_image_path}")
+        guide_images = []
+        for p in args.guide_image_path:
+            guide_images.extend(load_images(p))
+
+        print(f"loaded {len(guide_images)} guide images for guidance")
+        if len(guide_images) == 0:
+            print(f"No guide image, use previous generated image. / ガイド画像がありません。直前に生成した画像を使います: {args.image_path}")
+            guide_images = None
+    else:
+        guide_images = None
+
+    # seed指定時はseedを決めておく
+    if args.seed is not None:
+        # dynamic promptを使うと足りなくなる→images_per_promptを適当に大きくしておいてもらう
+        random.seed(args.seed)
+        predefined_seeds = [random.randint(0, 0x7FFFFFFF) for _ in range(args.n_iter * len(prompt_list) * args.images_per_prompt)]
+        if len(predefined_seeds) == 1:
+            predefined_seeds[0] = args.seed
+    else:
+        predefined_seeds = None
+
+    # デフォルト画像サイズを設定する：img2imgではこれらの値は無視される（またはW*Hにリサイズ済み）
+    if args.W is None:
+        args.W = 1024
+    if args.H is None:
+        args.H = 1024
+
+    # 画像生成のループ
+    os.makedirs(args.outdir, exist_ok=True)
+    max_embeddings_multiples = 1 if args.max_embeddings_multiples is None else args.max_embeddings_multiples
+
+    for gen_iter in range(args.n_iter):
+        print(f"iteration {gen_iter+1}/{args.n_iter}")
+        iter_seed = random.randint(0, 0x7FFFFFFF)
+
+        # バッチ処理の関数
+        def process_batch(batch: List[BatchData], highres_fix, highres_1st=False):
+            batch_size = len(batch)
+
+            # highres_fixの処理
+            if highres_fix and not highres_1st:
+                # 1st stageのバッチを作成して呼び出す：サイズを小さくして呼び出す
+                is_1st_latent = upscaler.support_latents() if upscaler else args.highres_fix_latents_upscaling
+
+                print("process 1st stage")
+                batch_1st = []
+                for _, base, ext in batch:
+
+                    def scale_and_round(x):
+                        if x is None:
+                            return None
+                        return int(x * args.highres_fix_scale + 0.5)
+
+                    width_1st = scale_and_round(ext.width)
+                    height_1st = scale_and_round(ext.height)
+                    width_1st = width_1st - width_1st % 32
+                    height_1st = height_1st - height_1st % 32
+
+                    original_width_1st = scale_and_round(ext.original_width)
+                    original_height_1st = scale_and_round(ext.original_height)
+                    crop_left_1st = scale_and_round(ext.crop_left)
+                    crop_top_1st = scale_and_round(ext.crop_top)
+
+                    strength_1st = ext.strength if args.highres_fix_strength is None else args.highres_fix_strength
+
+                    ext_1st = BatchDataExt(
+                        width_1st,
+                        height_1st,
+                        original_width_1st,
+                        original_height_1st,
+                        crop_left_1st,
+                        crop_top_1st,
+                        args.highres_fix_steps,
+                        ext.scale,
+                        ext.negative_scale,
+                        strength_1st,
+                        ext.network_muls,
+                        ext.num_sub_prompts,
+                    )
+                    batch_1st.append(BatchData(is_1st_latent, base, ext_1st))
+
+                pipe.set_enable_control_net(True)  # 1st stageではControlNetを有効にする
+                images_1st = process_batch(batch_1st, True, True)
+
+                # 2nd stageのバッチを作成して以下処理する
+                print("process 2nd stage")
+                width_2nd, height_2nd = batch[0].ext.width, batch[0].ext.height
+
+                if upscaler:
+                    # upscalerを使って画像を拡大する
+                    lowreso_imgs = None if is_1st_latent else images_1st
+                    lowreso_latents = None if not is_1st_latent else images_1st
+
+                    # 戻り値はPIL.Image.Imageかtorch.Tensorのlatents
+                    batch_size = len(images_1st)
+                    vae_batch_size = (
+                        batch_size
+                        if args.vae_batch_size is None
+                        else (max(1, int(batch_size * args.vae_batch_size)) if args.vae_batch_size < 1 else args.vae_batch_size)
+                    )
+                    vae_batch_size = int(vae_batch_size)
+                    images_1st = upscaler.upscale(
+                        vae, lowreso_imgs, lowreso_latents, dtype, width_2nd, height_2nd, batch_size, vae_batch_size
+                    )
+
+                elif args.highres_fix_latents_upscaling:
+                    # latentを拡大する
+                    org_dtype = images_1st.dtype
+                    if images_1st.dtype == torch.bfloat16:
+                        images_1st = images_1st.to(torch.float)  # interpolateがbf16をサポートしていない
+                    images_1st = torch.nn.functional.interpolate(
+                        images_1st, (batch[0].ext.height // 8, batch[0].ext.width // 8), mode="bilinear"
+                    )  # , antialias=True)
+                    images_1st = images_1st.to(org_dtype)
+
+                else:
+                    # 画像をLANCZOSで拡大する
+                    images_1st = [image.resize((width_2nd, height_2nd), resample=PIL.Image.LANCZOS) for image in images_1st]
+
+                batch_2nd = []
+                for i, (bd, image) in enumerate(zip(batch, images_1st)):
+                    bd_2nd = BatchData(False, BatchDataBase(*bd.base[0:3], bd.base.seed + 1, image, None, *bd.base[6:]), bd.ext)
+                    batch_2nd.append(bd_2nd)
+                batch = batch_2nd
+
+                if args.highres_fix_disable_control_net:
+                    pipe.set_enable_control_net(False)  # オプション指定時、2nd stageではControlNetを無効にする
+
+            # このバッチの情報を取り出す
+            (
+                return_latents,
+                (step_first, _, _, _, init_image, mask_image, _, guide_image),
+                (
+                    width,
+                    height,
+                    original_width,
+                    original_height,
+                    crop_left,
+                    crop_top,
+                    steps,
+                    scale,
+                    negative_scale,
+                    strength,
+                    network_muls,
+                    num_sub_prompts,
+                ),
+            ) = batch[0]
+            noise_shape = (LATENT_CHANNELS, height // DOWNSAMPLING_FACTOR, width // DOWNSAMPLING_FACTOR)
+
+            prompts = []
+            negative_prompts = []
+            start_code = torch.zeros((batch_size, *noise_shape), device=device, dtype=dtype)
+            noises = [
+                torch.zeros((batch_size, *noise_shape), device=device, dtype=dtype)
+                for _ in range(steps * scheduler_num_noises_per_step)
+            ]
+            seeds = []
+            clip_prompts = []
+
+            if init_image is not None:  # img2img?
+                i2i_noises = torch.zeros((batch_size, *noise_shape), device=device, dtype=dtype)
+                init_images = []
+
+                if mask_image is not None:
+                    mask_images = []
+                else:
+                    mask_images = None
+            else:
+                i2i_noises = None
+                init_images = None
+                mask_images = None
+
+            if guide_image is not None:  # CLIP image guided?
+                guide_images = []
+            else:
+                guide_images = None
+
+            # バッチ内の位置に関わらず同じ乱数を使うためにここで乱数を生成しておく。あわせてimage/maskがbatch内で同一かチェックする
+            all_images_are_same = True
+            all_masks_are_same = True
+            all_guide_images_are_same = True
+            for i, (_, (_, prompt, negative_prompt, seed, init_image, mask_image, clip_prompt, guide_image), _) in enumerate(batch):
+                prompts.append(prompt)
+                negative_prompts.append(negative_prompt)
+                seeds.append(seed)
+                clip_prompts.append(clip_prompt)
+
+                if init_image is not None:
+                    init_images.append(init_image)
+                    if i > 0 and all_images_are_same:
+                        all_images_are_same = init_images[-2] is init_image
+
+                if mask_image is not None:
+                    mask_images.append(mask_image)
+                    if i > 0 and all_masks_are_same:
+                        all_masks_are_same = mask_images[-2] is mask_image
+
+                if guide_image is not None:
+                    if type(guide_image) is list:
+                        guide_images.extend(guide_image)
+                        all_guide_images_are_same = False
+                    else:
+                        guide_images.append(guide_image)
+                        if i > 0 and all_guide_images_are_same:
+                            all_guide_images_are_same = guide_images[-2] is guide_image
+
+                # make start code
+                torch.manual_seed(seed)
+                start_code[i] = torch.randn(noise_shape, device=device, dtype=dtype)
+
+                # make each noises
+                for j in range(steps * scheduler_num_noises_per_step):
+                    noises[j][i] = torch.randn(noise_shape, device=device, dtype=dtype)
+
+                if i2i_noises is not None:  # img2img noise
+                    i2i_noises[i] = torch.randn(noise_shape, device=device, dtype=dtype)
+
+            noise_manager.reset_sampler_noises(noises)
+
+            # すべての画像が同じなら1枚だけpipeに渡すことでpipe側で処理を高速化する
+            if init_images is not None and all_images_are_same:
+                init_images = init_images[0]
+            if mask_images is not None and all_masks_are_same:
+                mask_images = mask_images[0]
+            if guide_images is not None and all_guide_images_are_same:
+                guide_images = guide_images[0]
+
+            # ControlNet使用時はguide imageをリサイズする
+            if control_nets:
+                # TODO resampleのメソッド
+                guide_images = guide_images if type(guide_images) == list else [guide_images]
+                guide_images = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in guide_images]
+                if len(guide_images) == 1:
+                    guide_images = guide_images[0]
+
+            # generate
+            if networks:
+                # 追加ネットワークの処理
+                shared = {}
+                for n, m in zip(networks, network_muls if network_muls else network_default_muls):
+                    n.set_multiplier(m)
+                    if regional_network:
+                        n.set_current_generation(batch_size, num_sub_prompts, width, height, shared)
+
+                if not regional_network and network_pre_calc:
+                    for n in networks:
+                        n.restore_weights()
+                    for n in networks:
+                        n.pre_calculation()
+                    print("pre-calculation... done")
+
+            images = pipe(
+                prompts,
+                negative_prompts,
+                init_images,
+                mask_images,
+                height,
+                width,
+                original_height,
+                original_width,
+                crop_top,
+                crop_left,
+                steps,
+                scale,
+                negative_scale,
+                strength,
+                latents=start_code,
+                output_type="pil",
+                max_embeddings_multiples=max_embeddings_multiples,
+                img2img_noise=i2i_noises,
+                vae_batch_size=args.vae_batch_size,
+                return_latents=return_latents,
+                clip_prompts=clip_prompts,
+                clip_guide_images=guide_images,
+            )
+            if highres_1st and not args.highres_fix_save_1st:  # return images or latents
+                return images
+
+            # save image
+            highres_prefix = ("0" if highres_1st else "1") if highres_fix else ""
+            ts_str = time.strftime("%Y%m%d%H%M%S", time.localtime())
+            for i, (image, prompt, negative_prompts, seed, clip_prompt) in enumerate(
+                zip(images, prompts, negative_prompts, seeds, clip_prompts)
+            ):
+                metadata = PngInfo()
+                metadata.add_text("prompt", prompt)
+                metadata.add_text("seed", str(seed))
+                metadata.add_text("sampler", args.sampler)
+                metadata.add_text("steps", str(steps))
+                metadata.add_text("scale", str(scale))
+                if negative_prompt is not None:
+                    metadata.add_text("negative-prompt", negative_prompt)
+                if negative_scale is not None:
+                    metadata.add_text("negative-scale", str(negative_scale))
+                if clip_prompt is not None:
+                    metadata.add_text("clip-prompt", clip_prompt)
+                metadata.add_text("original-height", str(original_height))
+                metadata.add_text("original-width", str(original_width))
+                metadata.add_text("crop-top", str(crop_top))
+                metadata.add_text("crop-left", str(crop_left))
+
+                if args.use_original_file_name and init_images is not None:
+                    if type(init_images) is list:
+                        fln = os.path.splitext(os.path.basename(init_images[i % len(init_images)].filename))[0] + ".png"
+                    else:
+                        fln = os.path.splitext(os.path.basename(init_images.filename))[0] + ".png"
+                elif args.sequential_file_name:
+                    fln = f"im_{highres_prefix}{step_first + i + 1:06d}.png"
+                else:
+                    fln = f"im_{ts_str}_{highres_prefix}{i:03d}_{seed}.png"
+
+                image.save(os.path.join(args.outdir, fln), pnginfo=metadata)
+
+            if not args.no_preview and not highres_1st and args.interactive:
+                try:
+                    import cv2
+
+                    for prompt, image in zip(prompts, images):
+                        cv2.imshow(prompt[:128], np.array(image)[:, :, ::-1])  # プロンプトが長いと死ぬ
+                        cv2.waitKey()
+                        cv2.destroyAllWindows()
+                except ImportError:
+                    print("opencv-python is not installed, cannot preview / opencv-pythonがインストールされていないためプレビューできません")
+
+            return images
+
+        # 画像生成のプロンプトが一周するまでのループ
+        prompt_index = 0
+        global_step = 0
+        batch_data = []
+        while args.interactive or prompt_index < len(prompt_list):
+            if len(prompt_list) == 0:
+                # interactive
+                valid = False
+                while not valid:
+                    print("\nType prompt:")
+                    try:
+                        raw_prompt = input()
+                    except EOFError:
+                        break
+
+                    valid = len(raw_prompt.strip().split(" --")[0].strip()) > 0
+                if not valid:  # EOF, end app
+                    break
+            else:
+                raw_prompt = prompt_list[prompt_index]
+
+            # sd-dynamic-prompts like variants:
+            # count is 1 (not dynamic) or images_per_prompt (no enumeration) or arbitrary (enumeration)
+            raw_prompts = handle_dynamic_prompt_variants(raw_prompt, args.images_per_prompt)
+
+            # repeat prompt
+            for pi in range(args.images_per_prompt if len(raw_prompts) == 1 else len(raw_prompts)):
+                raw_prompt = raw_prompts[pi] if len(raw_prompts) > 1 else raw_prompts[0]
+
+                if pi == 0 or len(raw_prompts) > 1:
+                    # parse prompt: if prompt is not changed, skip parsing
+                    width = args.W
+                    height = args.H
+                    original_width = args.original_width
+                    original_height = args.original_height
+                    crop_top = args.crop_top
+                    crop_left = args.crop_left
+                    scale = args.scale
+                    negative_scale = args.negative_scale
+                    steps = args.steps
+                    seed = None
+                    seeds = None
+                    strength = 0.8 if args.strength is None else args.strength
+                    negative_prompt = ""
+                    clip_prompt = None
+                    network_muls = None
+
+                    prompt_args = raw_prompt.strip().split(" --")
+                    prompt = prompt_args[0]
+                    print(f"prompt {prompt_index+1}/{len(prompt_list)}: {prompt}")
+
+                    for parg in prompt_args[1:]:
+                        try:
+                            m = re.match(r"w (\d+)", parg, re.IGNORECASE)
+                            if m:
+                                width = int(m.group(1))
+                                print(f"width: {width}")
+                                continue
+
+                            m = re.match(r"h (\d+)", parg, re.IGNORECASE)
+                            if m:
+                                height = int(m.group(1))
+                                print(f"height: {height}")
+                                continue
+
+                            m = re.match(r"ow (\d+)", parg, re.IGNORECASE)
+                            if m:
+                                original_width = int(m.group(1))
+                                print(f"original width: {width}")
+                                continue
+
+                            m = re.match(r"oh (\d+)", parg, re.IGNORECASE)
+                            if m:
+                                original_height = int(m.group(1))
+                                print(f"original height: {height}")
+                                continue
+
+                            m = re.match(r"ct (\d+)", parg, re.IGNORECASE)
+                            if m:
+                                crop_top = int(m.group(1))
+                                print(f"crop top: {crop_top}")
+                                continue
+
+                            m = re.match(r"cl (\d+)", parg, re.IGNORECASE)
+                            if m:
+                                crop_left = int(m.group(1))
+                                print(f"crop left: {crop_left}")
+                                continue
+
+                            m = re.match(r"s (\d+)", parg, re.IGNORECASE)
+                            if m:  # steps
+                                steps = max(1, min(1000, int(m.group(1))))
+                                print(f"steps: {steps}")
+                                continue
+
+                            m = re.match(r"d ([\d,]+)", parg, re.IGNORECASE)
+                            if m:  # seed
+                                seeds = [int(d) for d in m.group(1).split(",")]
+                                print(f"seeds: {seeds}")
+                                continue
+
+                            m = re.match(r"l ([\d\.]+)", parg, re.IGNORECASE)
+                            if m:  # scale
+                                scale = float(m.group(1))
+                                print(f"scale: {scale}")
+                                continue
+
+                            m = re.match(r"nl ([\d\.]+|none|None)", parg, re.IGNORECASE)
+                            if m:  # negative scale
+                                if m.group(1).lower() == "none":
+                                    negative_scale = None
+                                else:
+                                    negative_scale = float(m.group(1))
+                                print(f"negative scale: {negative_scale}")
+                                continue
+
+                            m = re.match(r"t ([\d\.]+)", parg, re.IGNORECASE)
+                            if m:  # strength
+                                strength = float(m.group(1))
+                                print(f"strength: {strength}")
+                                continue
+
+                            m = re.match(r"n (.+)", parg, re.IGNORECASE)
+                            if m:  # negative prompt
+                                negative_prompt = m.group(1)
+                                print(f"negative prompt: {negative_prompt}")
+                                continue
+
+                            m = re.match(r"c (.+)", parg, re.IGNORECASE)
+                            if m:  # clip prompt
+                                clip_prompt = m.group(1)
+                                print(f"clip prompt: {clip_prompt}")
+                                continue
+
+                            m = re.match(r"am ([\d\.\-,]+)", parg, re.IGNORECASE)
+                            if m:  # network multiplies
+                                network_muls = [float(v) for v in m.group(1).split(",")]
+                                while len(network_muls) < len(networks):
+                                    network_muls.append(network_muls[-1])
+                                print(f"network mul: {network_muls}")
+                                continue
+
+                        except ValueError as ex:
+                            print(f"Exception in parsing / 解析エラー: {parg}")
+                            print(ex)
+
+                # prepare seed
+                if seeds is not None:  # given in prompt
+                    # 数が足りないなら前のをそのまま使う
+                    if len(seeds) > 0:
+                        seed = seeds.pop(0)
+                else:
+                    if predefined_seeds is not None:
+                        if len(predefined_seeds) > 0:
+                            seed = predefined_seeds.pop(0)
+                        else:
+                            print("predefined seeds are exhausted")
+                            seed = None
+                    elif args.iter_same_seed:
+                        seeds = iter_seed
+                    else:
+                        seed = None  # 前のを消す
+
+                if seed is None:
+                    seed = random.randint(0, 0x7FFFFFFF)
+                if args.interactive:
+                    print(f"seed: {seed}")
+
+                # prepare init image, guide image and mask
+                init_image = mask_image = guide_image = None
+
+                # 同一イメージを使うとき、本当はlatentに変換しておくと無駄がないが面倒なのでとりあえず毎回処理する
+                if init_images is not None:
+                    init_image = init_images[global_step % len(init_images)]
+
+                    # img2imgの場合は、基本的に元画像のサイズで生成する。highres fixの場合はargs.W, args.Hとscaleに従いリサイズ済みなので無視する
+                    # 32単位に丸めたやつにresizeされるので踏襲する
+                    if not highres_fix:
+                        width, height = init_image.size
+                        width = width - width % 32
+                        height = height - height % 32
+                        if width != init_image.size[0] or height != init_image.size[1]:
+                            print(
+                                f"img2img image size is not divisible by 32 so aspect ratio is changed / img2imgの画像サイズが32で割り切れないためリサイズされます。画像が歪みます"
+                            )
+
+                if mask_images is not None:
+                    mask_image = mask_images[global_step % len(mask_images)]
+
+                if guide_images is not None:
+                    if control_nets:  # 複数件の場合あり
+                        c = len(control_nets)
+                        p = global_step % (len(guide_images) // c)
+                        guide_image = guide_images[p * c : p * c + c]
+                    else:
+                        guide_image = guide_images[global_step % len(guide_images)]
+
+                if regional_network:
+                    num_sub_prompts = len(prompt.split(" AND "))
+                    assert (
+                        len(networks) <= num_sub_prompts
+                    ), "Number of networks must be less than or equal to number of sub prompts."
+                else:
+                    num_sub_prompts = None
+
+                b1 = BatchData(
+                    False,
+                    BatchDataBase(global_step, prompt, negative_prompt, seed, init_image, mask_image, clip_prompt, guide_image),
+                    BatchDataExt(
+                        width,
+                        height,
+                        original_width,
+                        original_height,
+                        crop_left,
+                        crop_top,
+                        steps,
+                        scale,
+                        negative_scale,
+                        strength,
+                        tuple(network_muls) if network_muls else None,
+                        num_sub_prompts,
+                    ),
+                )
+                if len(batch_data) > 0 and batch_data[-1].ext != b1.ext:  # バッチ分割必要？
+                    process_batch(batch_data, highres_fix)
+                    batch_data.clear()
+
+                batch_data.append(b1)
+                if len(batch_data) == args.batch_size:
+                    prev_image = process_batch(batch_data, highres_fix)[0]
+                    batch_data.clear()
+
+                global_step += 1
+
+            prompt_index += 1
+
+        if len(batch_data) > 0:
+            process_batch(batch_data, highres_fix)
+            batch_data.clear()
+
+    print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--prompt", type=str, default=None, help="prompt / プロンプト")
+    parser.add_argument(
+        "--from_file", type=str, default=None, help="if specified, load prompts from this file / 指定時はプロンプトをファイルから読み込む"
+    )
+    parser.add_argument(
+        "--interactive", action="store_true", help="interactive mode (generates one image) / 対話モード（生成される画像は1枚になります）"
+    )
+    parser.add_argument(
+        "--no_preview", action="store_true", help="do not show generated image in interactive mode / 対話モードで画像を表示しない"
+    )
+    parser.add_argument(
+        "--image_path", type=str, default=None, help="image to inpaint or to generate from / img2imgまたはinpaintを行う元画像"
+    )
+    parser.add_argument("--mask_path", type=str, default=None, help="mask in inpainting / inpaint時のマスク")
+    parser.add_argument("--strength", type=float, default=None, help="img2img strength / img2img時のstrength")
+    parser.add_argument("--images_per_prompt", type=int, default=1, help="number of images per prompt / プロンプトあたりの出力枚数")
+    parser.add_argument("--outdir", type=str, default="outputs", help="dir to write results to / 生成画像の出力先")
+    parser.add_argument("--sequential_file_name", action="store_true", help="sequential output file name / 生成画像のファイル名を連番にする")
+    parser.add_argument(
+        "--use_original_file_name",
+        action="store_true",
+        help="prepend original file name in img2img / img2imgで元画像のファイル名を生成画像のファイル名の先頭に付ける",
+    )
+    # parser.add_argument("--ddim_eta", type=float, default=0.0, help="ddim eta (eta=0.0 corresponds to deterministic sampling", )
+    parser.add_argument("--n_iter", type=int, default=1, help="sample this often / 繰り返し回数")
+    parser.add_argument("--H", type=int, default=None, help="image height, in pixel space / 生成画像高さ")
+    parser.add_argument("--W", type=int, default=None, help="image width, in pixel space / 生成画像幅")
+    parser.add_argument(
+        "--original_height", type=int, default=None, help="original height for SDXL conditioning / SDXLの条件付けに用いるoriginal heightの値"
+    )
+    parser.add_argument(
+        "--original_width", type=int, default=None, help="original width for SDXL conditioning / SDXLの条件付けに用いるoriginal widthの値"
+    )
+    parser.add_argument("--crop_top", type=int, default=None, help="crop top for SDXL conditioning / SDXLの条件付けに用いるcrop topの値")
+    parser.add_argument("--crop_left", type=int, default=None, help="crop left for SDXL conditioning / SDXLの条件付けに用いるcrop leftの値")
+    parser.add_argument("--batch_size", type=int, default=1, help="batch size / バッチサイズ")
+    parser.add_argument(
+        "--vae_batch_size",
+        type=float,
+        default=None,
+        help="batch size for VAE, < 1.0 for ratio / VAE処理時のバッチサイズ、1未満の値の場合は通常バッチサイズの比率",
+    )
+    parser.add_argument(
+        "--vae_slices",
+        type=int,
+        default=None,
+        help="number of slices to split image into for VAE to reduce VRAM usage, None for no splitting (default), slower if specified. 16 or 32 recommended / VAE処理時にVRAM使用量削減のため画像を分割するスライス数、Noneの場合は分割しない（デフォルト）、指定すると遅くなる。16か32程度を推奨",
+    )
+    parser.add_argument("--no_half_vae", action="store_true", help="do not use fp16/bf16 precision for VAE / VAE処理時にfp16/bf16を使わない")
+    parser.add_argument("--steps", type=int, default=50, help="number of ddim sampling steps / サンプリングステップ数")
+    parser.add_argument(
+        "--sampler",
+        type=str,
+        default="ddim",
+        choices=[
+            "ddim",
+            "pndm",
+            "lms",
+            "euler",
+            "euler_a",
+            "heun",
+            "dpm_2",
+            "dpm_2_a",
+            "dpmsolver",
+            "dpmsolver++",
+            "dpmsingle",
+            "k_lms",
+            "k_euler",
+            "k_euler_a",
+            "k_dpm_2",
+            "k_dpm_2_a",
+        ],
+        help=f"sampler (scheduler) type / サンプラー（スケジューラ）の種類",
+    )
+    parser.add_argument(
+        "--scale",
+        type=float,
+        default=7.5,
+        help="unconditional guidance scale: eps = eps(x, empty) + scale * (eps(x, cond) - eps(x, empty)) / guidance scale",
+    )
+    parser.add_argument("--ckpt", type=str, default=None, help="path to checkpoint of model / モデルのcheckpointファイルまたはディレクトリ")
+    parser.add_argument(
+        "--vae", type=str, default=None, help="path to checkpoint of vae to replace / VAEを入れ替える場合、VAEのcheckpointファイルまたはディレクトリ"
+    )
+    parser.add_argument(
+        "--tokenizer_cache_dir",
+        type=str,
+        default=None,
+        help="directory for caching Tokenizer (for offline training) / Tokenizerをキャッシュするディレクトリ（ネット接続なしでの学習のため）",
+    )
+    # parser.add_argument("--replace_clip_l14_336", action='store_true',
+    #                     help="Replace CLIP (Text Encoder) to l/14@336 / CLIP(Text Encoder)をl/14@336に入れ替える")
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=None,
+        help="seed, or seed of seeds in multiple generation / 1枚生成時のseed、または複数枚生成時の乱数seedを決めるためのseed",
+    )
+    parser.add_argument(
+        "--iter_same_seed",
+        action="store_true",
+        help="use same seed for all prompts in iteration if no seed specified / 乱数seedの指定がないとき繰り返し内はすべて同じseedを使う（プロンプト間の差異の比較用）",
+    )
+    parser.add_argument("--fp16", action="store_true", help="use fp16 / fp16を指定し省メモリ化する")
+    parser.add_argument("--bf16", action="store_true", help="use bfloat16 / bfloat16を指定し省メモリ化する")
+    parser.add_argument("--xformers", action="store_true", help="use xformers / xformersを使用し高速化する")
+    parser.add_argument("--sdpa", action="store_true", help="use sdpa in PyTorch 2 / sdpa")
+    parser.add_argument(
+        "--diffusers_xformers",
+        action="store_true",
+        help="use xformers by diffusers (Hypernetworks doesn't work) / Diffusersでxformersを使用する（Hypernetwork利用不可）",
+    )
+    parser.add_argument(
+        "--opt_channels_last", action="store_true", help="set channels last option to model / モデルにchannels lastを指定し最適化する"
+    )
+    parser.add_argument(
+        "--network_module", type=str, default=None, nargs="*", help="additional network module to use / 追加ネットワークを使う時そのモジュール名"
+    )
+    parser.add_argument(
+        "--network_weights", type=str, default=None, nargs="*", help="additional network weights to load / 追加ネットワークの重み"
+    )
+    parser.add_argument("--network_mul", type=float, default=None, nargs="*", help="additional network multiplier / 追加ネットワークの効果の倍率")
+    parser.add_argument(
+        "--network_args", type=str, default=None, nargs="*", help="additional argmuments for network (key=value) / ネットワークへの追加の引数"
+    )
+    parser.add_argument("--network_show_meta", action="store_true", help="show metadata of network model / ネットワークモデルのメタデータを表示する")
+    parser.add_argument("--network_merge", action="store_true", help="merge network weights to original model / ネットワークの重みをマージする")
+    parser.add_argument(
+        "--network_pre_calc", action="store_true", help="pre-calculate network for generation / ネットワークのあらかじめ計算して生成する"
+    )
+    parser.add_argument(
+        "--textual_inversion_embeddings",
+        type=str,
+        default=None,
+        nargs="*",
+        help="Embeddings files of Textual Inversion / Textual Inversionのembeddings",
+    )
+    parser.add_argument("--clip_skip", type=int, default=None, help="layer number from bottom to use in CLIP / CLIPの後ろからn層目の出力を使う")
+    parser.add_argument(
+        "--max_embeddings_multiples",
+        type=int,
+        default=None,
+        help="max embeding multiples, max token length is 75 * multiples / トークン長をデフォルトの何倍とするか 75*この値 がトークン長となる",
+    )
+    parser.add_argument(
+        "--guide_image_path", type=str, default=None, nargs="*", help="image to CLIP guidance / CLIP guided SDでガイドに使う画像"
+    )
+    parser.add_argument(
+        "--highres_fix_scale",
+        type=float,
+        default=None,
+        help="enable highres fix, reso scale for 1st stage / highres fixを有効にして最初の解像度をこのscaleにする",
+    )
+    parser.add_argument(
+        "--highres_fix_steps", type=int, default=28, help="1st stage steps for highres fix / highres fixの最初のステージのステップ数"
+    )
+    parser.add_argument(
+        "--highres_fix_strength",
+        type=float,
+        default=None,
+        help="1st stage img2img strength for highres fix / highres fixの最初のステージのimg2img時のstrength、省略時はstrengthと同じ",
+    )
+    parser.add_argument(
+        "--highres_fix_save_1st", action="store_true", help="save 1st stage images for highres fix / highres fixの最初のステージの画像を保存する"
+    )
+    parser.add_argument(
+        "--highres_fix_latents_upscaling",
+        action="store_true",
+        help="use latents upscaling for highres fix / highres fixでlatentで拡大する",
+    )
+    parser.add_argument(
+        "--highres_fix_upscaler", type=str, default=None, help="upscaler module for highres fix / highres fixで使うupscalerのモジュール名"
+    )
+    parser.add_argument(
+        "--highres_fix_upscaler_args",
+        type=str,
+        default=None,
+        help="additional argmuments for upscaler (key=value) / upscalerへの追加の引数",
+    )
+    parser.add_argument(
+        "--highres_fix_disable_control_net",
+        action="store_true",
+        help="disable ControlNet for highres fix / highres fixでControlNetを使わない",
+    )
+
+    parser.add_argument(
+        "--negative_scale", type=float, default=None, help="set another guidance scale for negative prompt / ネガティブプロンプトのscaleを指定する"
+    )
+
+    parser.add_argument(
+        "--control_net_models", type=str, default=None, nargs="*", help="ControlNet models to use / 使用するControlNetのモデル名"
+    )
+    parser.add_argument(
+        "--control_net_preps", type=str, default=None, nargs="*", help="ControlNet preprocess to use / 使用するControlNetのプリプロセス名"
+    )
+    parser.add_argument("--control_net_weights", type=float, default=None, nargs="*", help="ControlNet weights / ControlNetの重み")
+    parser.add_argument(
+        "--control_net_ratios",
+        type=float,
+        default=None,
+        nargs="*",
+        help="ControlNet guidance ratio for steps / ControlNetでガイドするステップ比率",
+    )
+    # parser.add_argument(
+    #     "--control_net_image_path", type=str, default=None, nargs="*", help="image for ControlNet guidance / ControlNetでガイドに使う画像"
+    # )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    main(args)
diff --git a/sdxl_minimal_inference.py b/sdxl_minimal_inference.py
new file mode 100644
index 0000000000000000000000000000000000000000..25b8e51bfeb6269d43a862b6f458e43ff10606f4
--- /dev/null
+++ b/sdxl_minimal_inference.py
@@ -0,0 +1,313 @@
+# 手元で推論を行うための最低限のコード。HuggingFace／DiffusersのCLIP、schedulerとVAEを使う
+# Minimal code for performing inference at local. Use HuggingFace/Diffusers CLIP, scheduler and VAE
+
+import argparse
+import datetime
+import math
+import os
+import random
+from einops import repeat
+import numpy as np
+import torch
+from tqdm import tqdm
+from transformers import CLIPTokenizer
+from diffusers import EulerDiscreteScheduler
+from PIL import Image
+import open_clip
+from safetensors.torch import load_file
+
+from library import model_util, sdxl_model_util
+import networks.lora as lora
+
+# scheduler: このあたりの設定はSD1/2と同じでいいらしい
+# scheduler: The settings around here seem to be the same as SD1/2
+SCHEDULER_LINEAR_START = 0.00085
+SCHEDULER_LINEAR_END = 0.0120
+SCHEDULER_TIMESTEPS = 1000
+SCHEDLER_SCHEDULE = "scaled_linear"
+
+
+# Time EmbeddingはDiffusersからのコピー
+# Time Embedding is copied from Diffusers
+
+
+def timestep_embedding(timesteps, dim, max_period=10000, repeat_only=False):
+    """
+    Create sinusoidal timestep embeddings.
+    :param timesteps: a 1-D Tensor of N indices, one per batch element.
+                      These may be fractional.
+    :param dim: the dimension of the output.
+    :param max_period: controls the minimum frequency of the embeddings.
+    :return: an [N x dim] Tensor of positional embeddings.
+    """
+    if not repeat_only:
+        half = dim // 2
+        freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to(
+            device=timesteps.device
+        )
+        args = timesteps[:, None].float() * freqs[None]
+        embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+        if dim % 2:
+            embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
+    else:
+        embedding = repeat(timesteps, "b -> b d", d=dim)
+    return embedding
+
+
+def get_timestep_embedding(x, outdim):
+    assert len(x.shape) == 2
+    b, dims = x.shape[0], x.shape[1]
+    # x = rearrange(x, "b d -> (b d)")
+    x = torch.flatten(x)
+    emb = timestep_embedding(x, outdim)
+    # emb = rearrange(emb, "(b d) d2 -> b (d d2)", b=b, d=dims, d2=outdim)
+    emb = torch.reshape(emb, (b, dims * outdim))
+    return emb
+
+
+if __name__ == "__main__":
+    # 画像生成条件を変更する場合はここを変更 / change here to change image generation conditions
+
+    # SDXLの追加のvector embeddingへ渡す値 / Values to pass to additional vector embedding of SDXL
+    target_height = 1024
+    target_width = 1024
+    original_height = target_height
+    original_width = target_width
+    crop_top = 0
+    crop_left = 0
+
+    steps = 50
+    guidance_scale = 7
+    seed = None  # 1
+
+    DEVICE = "cuda"
+    DTYPE = torch.float16  # bfloat16 may work
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--ckpt_path", type=str, required=True)
+    parser.add_argument("--prompt", type=str, default="A photo of a cat")
+    parser.add_argument("--negative_prompt", type=str, default="")
+    parser.add_argument("--output_dir", type=str, default=".")
+    parser.add_argument(
+        "--lora_weights",
+        type=str,
+        nargs="*",
+        default=[],
+        help="LoRA weights, only supports networks.lora, each arguement is a `path;multiplier` (semi-colon separated)",
+    )
+    parser.add_argument("--interactive", action="store_true")
+    args = parser.parse_args()
+
+    # HuggingFaceのmodel id
+    text_encoder_1_name = "openai/clip-vit-large-patch14"
+    text_encoder_2_name = "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k"
+
+    # checkpointを読み込む。モデル変換についてはそちらの関数を参照
+    # Load checkpoint. For model conversion, see this function
+
+    # 本体RAMが少ない場合はGPUにロードするといいかも
+    # If the main RAM is small, it may be better to load it on the GPU
+    text_model1, text_model2, vae, unet, _, _ = sdxl_model_util.load_models_from_sdxl_checkpoint(
+        sdxl_model_util.MODEL_VERSION_SDXL_BASE_V0_9, args.ckpt_path, "cpu"
+    )
+
+    # Text Encoder 1はSDXL本体でもHuggingFaceのものを使っている
+    # In SDXL, Text Encoder 1 is also using HuggingFace's
+
+    # Text Encoder 2はSDXL本体ではopen_clipを使っている
+    # それを使ってもいいが、SD2のDiffusers版に合わせる形で、HuggingFaceのものを使う
+    # 重みの変換コードはSD2とほぼ同じ
+    # In SDXL, Text Encoder 2 is using open_clip
+    # It's okay to use it, but to match the Diffusers version of SD2, use HuggingFace's
+    # The weight conversion code is almost the same as SD2
+
+    # VAEの構造はSDXLもSD1/2と同じだが、重みは異なるようだ。何より謎のscale値が違う
+    # fp16でNaNが出やすいようだ
+    # The structure of VAE is the same as SD1/2, but the weights seem to be different. Above all, the mysterious scale value is different.
+    # NaN seems to be more likely to occur in fp16
+
+    unet.to(DEVICE, dtype=DTYPE)
+    unet.eval()
+
+    vae_dtype = DTYPE
+    if DTYPE == torch.float16:
+        print("use float32 for vae")
+        vae_dtype = torch.float32
+    vae.to(DEVICE, dtype=vae_dtype)
+    vae.eval()
+
+    text_model1.to(DEVICE, dtype=DTYPE)
+    text_model1.eval()
+    text_model2.to(DEVICE, dtype=DTYPE)
+    text_model2.eval()
+
+    unet.set_use_memory_efficient_attention(True, False)
+    vae.set_use_memory_efficient_attention_xformers(True)
+
+    # Tokenizers
+    tokenizer1 = CLIPTokenizer.from_pretrained(text_encoder_1_name)
+    tokenizer2 = lambda x: open_clip.tokenize(x, context_length=77)
+
+    # LoRA
+    for weights_file in args.lora_weights:
+        if ";" in weights_file:
+            weights_file, multiplier = weights_file.split(";")
+            multiplier = float(multiplier)
+        else:
+            multiplier = 1.0
+
+        lora_model, weights_sd = lora.create_network_from_weights(
+            multiplier, weights_file, vae, [text_model1, text_model2], unet, None, True
+        )
+        lora_model.merge_to([text_model1, text_model2], unet, weights_sd, DTYPE, DEVICE)
+
+    # scheduler
+    scheduler = EulerDiscreteScheduler(
+        num_train_timesteps=SCHEDULER_TIMESTEPS,
+        beta_start=SCHEDULER_LINEAR_START,
+        beta_end=SCHEDULER_LINEAR_END,
+        beta_schedule=SCHEDLER_SCHEDULE,
+    )
+
+    def generate_image(prompt, negative_prompt, seed=None):
+        # 将来的にサイズ情報も変えられるようにする / Make it possible to change the size information in the future
+        # prepare embedding
+        with torch.no_grad():
+            # vector
+            emb1 = get_timestep_embedding(torch.FloatTensor([original_height, original_width]).unsqueeze(0), 256)
+            emb2 = get_timestep_embedding(torch.FloatTensor([crop_top, crop_left]).unsqueeze(0), 256)
+            emb3 = get_timestep_embedding(torch.FloatTensor([target_height, target_width]).unsqueeze(0), 256)
+            # print("emb1", emb1.shape)
+            c_vector = torch.cat([emb1, emb2, emb3], dim=1).to(DEVICE, dtype=DTYPE)
+            uc_vector = c_vector.clone().to(DEVICE, dtype=DTYPE)  # ちょっとここ正しいかどうかわからない I'm not sure if this is right
+
+            # crossattn
+
+        # Text Encoderを二つ呼ぶ関数  Function to call two Text Encoders
+        def call_text_encoder(text):
+            # text encoder 1
+            batch_encoding = tokenizer1(
+                text,
+                truncation=True,
+                return_length=True,
+                return_overflowing_tokens=False,
+                padding="max_length",
+                return_tensors="pt",
+            )
+            tokens = batch_encoding["input_ids"].to(DEVICE)
+
+            with torch.no_grad():
+                enc_out = text_model1(tokens, output_hidden_states=True, return_dict=True)
+                text_embedding1 = enc_out["hidden_states"][11]
+                # text_embedding = pipe.text_encoder.text_model.final_layer_norm(text_embedding)    # layer normは通さないらしい
+
+            # text encoder 2
+            with torch.no_grad():
+                tokens = tokenizer2(text).to(DEVICE)
+
+                enc_out = text_model2(tokens, output_hidden_states=True, return_dict=True)
+                text_embedding2_penu = enc_out["hidden_states"][-2]
+                # print("hidden_states2", text_embedding2_penu.shape)
+                text_embedding2_pool = enc_out["text_embeds"]
+
+            # 連結して終了 concat and finish
+            text_embedding = torch.cat([text_embedding1, text_embedding2_penu], dim=2)
+            return text_embedding, text_embedding2_pool
+
+        # cond
+        c_ctx, c_ctx_pool = call_text_encoder(prompt)
+        # print(c_ctx.shape, c_ctx_p.shape, c_vector.shape)
+        c_vector = torch.cat([c_ctx_pool, c_vector], dim=1)
+
+        # uncond
+        uc_ctx, uc_ctx_pool = call_text_encoder(negative_prompt)
+        uc_vector = torch.cat([uc_ctx_pool, uc_vector], dim=1)
+
+        text_embeddings = torch.cat([uc_ctx, c_ctx])
+        vector_embeddings = torch.cat([uc_vector, c_vector])
+
+        # メモリ使用量を減らすにはここでText Encoderを削除するかCPUへ移動する
+
+        if seed is not None:
+            random.seed(seed)
+            np.random.seed(seed)
+            torch.manual_seed(seed)
+            torch.cuda.manual_seed_all(seed)
+
+            # # random generator for initial noise
+            # generator = torch.Generator(device="cuda").manual_seed(seed)
+            generator = None
+        else:
+            generator = None
+
+        # get the initial random noise unless the user supplied it
+        # SDXLはCPUでlatentsを作成しているので一応合わせておく、Diffusersはtarget deviceでlatentsを作成している
+        # SDXL creates latents in CPU, Diffusers creates latents in target device
+        latents_shape = (1, 4, target_height // 8, target_width // 8)
+        latents = torch.randn(
+            latents_shape,
+            generator=generator,
+            device="cpu",
+            dtype=torch.float32,
+        ).to(DEVICE, dtype=DTYPE)
+
+        # scale the initial noise by the standard deviation required by the scheduler
+        latents = latents * scheduler.init_noise_sigma
+
+        # set timesteps
+        scheduler.set_timesteps(steps, DEVICE)
+
+        # このへんはDiffusersからのコピペ
+        # Copy from Diffusers
+        timesteps = scheduler.timesteps.to(DEVICE)  # .to(DTYPE)
+        num_latent_input = 2
+        with torch.no_grad():
+            for i, t in enumerate(tqdm(timesteps)):
+                # expand the latents if we are doing classifier free guidance
+                latent_model_input = latents.repeat((num_latent_input, 1, 1, 1))
+                latent_model_input = scheduler.scale_model_input(latent_model_input, t)
+
+                noise_pred = unet(latent_model_input, t, text_embeddings, vector_embeddings)
+
+                noise_pred_uncond, noise_pred_text = noise_pred.chunk(num_latent_input)  # uncond by negative prompt
+                noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+                # compute the previous noisy sample x_t -> x_t-1
+                # latents = scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
+                latents = scheduler.step(noise_pred, t, latents).prev_sample
+
+            # latents = 1 / 0.18215 * latents
+            latents = 1 / sdxl_model_util.VAE_SCALE_FACTOR * latents
+            latents = latents.to(vae_dtype)
+            image = vae.decode(latents).sample
+            image = (image / 2 + 0.5).clamp(0, 1)
+
+        # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16
+        image = image.cpu().permute(0, 2, 3, 1).float().numpy()
+
+        # image = self.numpy_to_pil(image)
+        image = (image * 255).round().astype("uint8")
+        image = [Image.fromarray(im) for im in image]
+
+        # 保存して終了 save and finish
+        timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+        for i, img in enumerate(image):
+            img.save(os.path.join(args.output_dir, f"image_{timestamp}_{i:03d}.png"))
+
+    if not args.interactive:
+        generate_image(args.prompt, args.negative_prompt, seed)
+    else:
+        # loop for interactive
+        while True:
+            prompt = input("prompt: ")
+            if prompt == "":
+                break
+            negative_prompt = input("negative prompt: ")
+            seed = input("seed: ")
+            if seed == "":
+                seed = None
+            else:
+                seed = int(seed)
+            generate_image(prompt, negative_prompt, seed)
+
+    print("Done!")
diff --git a/sdxl_train.py b/sdxl_train.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd5b74dda77c360a4f7c792fd6721791c500bfb1
--- /dev/null
+++ b/sdxl_train.py
@@ -0,0 +1,632 @@
+# training with captions
+
+import argparse
+import gc
+import math
+import os
+from multiprocessing import Value
+
+from tqdm import tqdm
+import torch
+from accelerate.utils import set_seed
+from diffusers import DDPMScheduler
+from library import sdxl_model_util
+
+import library.train_util as train_util
+import library.config_util as config_util
+import library.sdxl_train_util as sdxl_train_util
+from library.config_util import (
+    ConfigSanitizer,
+    BlueprintGenerator,
+)
+import library.custom_train_functions as custom_train_functions
+from library.custom_train_functions import (
+    apply_snr_weight,
+    prepare_scheduler_for_custom_training,
+    pyramid_noise_like,
+    apply_noise_offset,
+    scale_v_prediction_loss_like_noise_prediction,
+)
+from library.sdxl_original_unet import SdxlUNet2DConditionModel
+
+
+def train(args):
+    train_util.verify_training_args(args)
+    train_util.prepare_dataset_args(args, True)
+    sdxl_train_util.verify_sdxl_training_args(args)
+
+    assert not args.weighted_captions, "weighted_captions is not supported currently / weighted_captionsは現在サポートされていません"
+    assert (
+        not args.train_text_encoder or not args.cache_text_encoder_outputs
+    ), "cache_text_encoder_outputs is not supported when training text encoder / text encoderを学習するときはcache_text_encoder_outputsはサポートされていません"
+
+    cache_latents = args.cache_latents
+    use_dreambooth_method = args.in_json is None
+
+    if args.seed is not None:
+        set_seed(args.seed)  # 乱数系列を初期化する
+
+    tokenizer1, tokenizer2 = sdxl_train_util.load_tokenizers(args)
+
+    # データセットを準備する
+    if args.dataset_class is None:
+        blueprint_generator = BlueprintGenerator(ConfigSanitizer(True, True, False, True))
+        if args.dataset_config is not None:
+            print(f"Load dataset config from {args.dataset_config}")
+            user_config = config_util.load_user_config(args.dataset_config)
+            ignored = ["train_data_dir", "in_json"]
+            if any(getattr(args, attr) is not None for attr in ignored):
+                print(
+                    "ignore following options because config file is found: {0} / 設定ファイルが利用されるため以下のオプションは無視されます: {0}".format(
+                        ", ".join(ignored)
+                    )
+                )
+        else:
+            if use_dreambooth_method:
+                print("Using DreamBooth method.")
+                user_config = {
+                    "datasets": [
+                        {
+                            "subsets": config_util.generate_dreambooth_subsets_config_by_subdirs(
+                                args.train_data_dir, args.reg_data_dir
+                            )
+                        }
+                    ]
+                }
+            else:
+                print("Training with captions.")
+                user_config = {
+                    "datasets": [
+                        {
+                            "subsets": [
+                                {
+                                    "image_dir": args.train_data_dir,
+                                    "metadata_file": args.in_json,
+                                }
+                            ]
+                        }
+                    ]
+                }
+
+        blueprint = blueprint_generator.generate(user_config, args, tokenizer=[tokenizer1, tokenizer2])
+        train_dataset_group = config_util.generate_dataset_group_by_blueprint(blueprint.dataset_group)
+    else:
+        train_dataset_group = train_util.load_arbitrary_dataset(args, [tokenizer1, tokenizer2])
+
+    current_epoch = Value("i", 0)
+    current_step = Value("i", 0)
+    ds_for_collater = train_dataset_group if args.max_data_loader_n_workers == 0 else None
+    collater = train_util.collater_class(current_epoch, current_step, ds_for_collater)
+
+    if args.debug_dataset:
+        train_util.debug_dataset(train_dataset_group, True)
+        return
+    if len(train_dataset_group) == 0:
+        print(
+            "No data found. Please verify the metadata file and train_data_dir option. / 画像がありません。メタデータおよびtrain_data_dirオプションを確認してください。"
+        )
+        return
+
+    if cache_latents:
+        assert (
+            train_dataset_group.is_latent_cacheable()
+        ), "when caching latents, either color_aug or random_crop cannot be used / latentをキャッシュするときはcolor_augとrandom_cropは使えません"
+
+    if args.cache_text_encoder_outputs:
+        assert (
+            train_dataset_group.is_text_encoder_output_cacheable()
+        ), "when caching text encoder output, either caption_dropout_rate, shuffle_caption, token_warmup_step or caption_tag_dropout_rate cannot be used / text encoderの出力をキャッシュするときはcaption_dropout_rate, shuffle_caption, token_warmup_step, caption_tag_dropout_rateは使えません"
+
+    # acceleratorを準備する
+    print("prepare accelerator")
+    accelerator = train_util.prepare_accelerator(args)
+
+    # mixed precisionに対応した型を用意しておき適宜castする
+    weight_dtype, save_dtype = train_util.prepare_dtype(args)
+    vae_dtype = torch.float32 if args.no_half_vae else weight_dtype
+
+    # モデルを読み込む
+    (
+        load_stable_diffusion_format,
+        text_encoder1,
+        text_encoder2,
+        vae,
+        unet,
+        logit_scale,
+        ckpt_info,
+    ) = sdxl_train_util.load_target_model(args, accelerator, "sdxl", weight_dtype)
+    # logit_scale = logit_scale.to(accelerator.device, dtype=weight_dtype)
+
+    # verify load/save model formats
+    if load_stable_diffusion_format:
+        src_stable_diffusion_ckpt = args.pretrained_model_name_or_path
+        src_diffusers_model_path = None
+    else:
+        src_stable_diffusion_ckpt = None
+        src_diffusers_model_path = args.pretrained_model_name_or_path
+
+    if args.save_model_as is None:
+        save_stable_diffusion_format = load_stable_diffusion_format
+        use_safetensors = args.use_safetensors
+    else:
+        save_stable_diffusion_format = args.save_model_as.lower() == "ckpt" or args.save_model_as.lower() == "safetensors"
+        use_safetensors = args.use_safetensors or ("safetensors" in args.save_model_as.lower())
+        assert save_stable_diffusion_format, "save_model_as must be ckpt or safetensors / save_model_asはckptかsafetensorsである必要があります"
+
+    # Diffusers版のxformers使用フラグを設定する関数
+    def set_diffusers_xformers_flag(model, valid):
+        def fn_recursive_set_mem_eff(module: torch.nn.Module):
+            if hasattr(module, "set_use_memory_efficient_attention_xformers"):
+                module.set_use_memory_efficient_attention_xformers(valid)
+
+            for child in module.children():
+                fn_recursive_set_mem_eff(child)
+
+        fn_recursive_set_mem_eff(model)
+
+    # モデルに xformers とか memory efficient attention を組み込む
+    if args.diffusers_xformers:
+        # もうU-Netを独自にしたので動かないけどVAEのxformersは動くはず
+        accelerator.print("Use xformers by Diffusers")
+        # set_diffusers_xformers_flag(unet, True)
+        set_diffusers_xformers_flag(vae, True)
+    else:
+        # Windows版のxformersはfloatで学習できなかったりxformersを使わない設定も可能にしておく必要がある
+        accelerator.print("Disable Diffusers' xformers")
+        train_util.replace_unet_modules(unet, args.mem_eff_attn, args.xformers, args.sdpa)
+        vae.set_use_memory_efficient_attention_xformers(args.xformers)
+
+    # 学習を準備する
+    if cache_latents:
+        vae.to(accelerator.device, dtype=vae_dtype)
+        vae.requires_grad_(False)
+        vae.eval()
+        with torch.no_grad():
+            train_dataset_group.cache_latents(vae, args.vae_batch_size, args.cache_latents_to_disk, accelerator.is_main_process)
+        vae.to("cpu")
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        gc.collect()
+
+        accelerator.wait_for_everyone()
+
+    # 学習を準備する：モデルを適切な状態にする
+    training_models = []
+    if args.gradient_checkpointing:
+        unet.enable_gradient_checkpointing()
+    training_models.append(unet)
+
+    if args.train_text_encoder:
+        # TODO each option for two text encoders?
+        accelerator.print("enable text encoder training")
+        if args.gradient_checkpointing:
+            text_encoder1.gradient_checkpointing_enable()
+            text_encoder2.gradient_checkpointing_enable()
+        training_models.append(text_encoder1)
+        training_models.append(text_encoder2)
+
+        text_encoder1_cache = None
+        text_encoder2_cache = None
+
+        # set require_grad=True later
+    else:
+        text_encoder1.requires_grad_(False)
+        text_encoder2.requires_grad_(False)
+        text_encoder1.eval()
+        text_encoder2.eval()
+
+        # TextEncoderの出力をキャッシュする
+        if args.cache_text_encoder_outputs:
+            # Text Encodes are eval and no grad
+            text_encoder1_cache, text_encoder2_cache = sdxl_train_util.cache_text_encoder_outputs(
+                args, accelerator, (tokenizer1, tokenizer2), (text_encoder1, text_encoder2), train_dataset_group, None
+            )
+            accelerator.wait_for_everyone()
+
+    if not cache_latents:
+        vae.requires_grad_(False)
+        vae.eval()
+        vae.to(accelerator.device, dtype=vae_dtype)
+
+    for m in training_models:
+        m.requires_grad_(True)
+    params = []
+    for m in training_models:
+        params.extend(m.parameters())
+    params_to_optimize = params
+
+    # calculate number of trainable parameters
+    n_params = 0
+    for p in params:
+        n_params += p.numel()
+    accelerator.print(f"number of models: {len(training_models)}")
+    accelerator.print(f"number of trainable parameters: {n_params}")
+
+    # 学習に必要なクラスを準備する
+    accelerator.print("prepare optimizer, data loader etc.")
+    _, _, optimizer = train_util.get_optimizer(args, trainable_params=params_to_optimize)
+
+    # dataloaderを準備する
+    # DataLoaderのプロセス数：0はメインプロセスになる
+    n_workers = min(args.max_data_loader_n_workers, os.cpu_count() - 1)  # cpu_count-1 ただし最大で指定された数まで
+    train_dataloader = torch.utils.data.DataLoader(
+        train_dataset_group,
+        batch_size=1,
+        shuffle=True,
+        collate_fn=collater,
+        num_workers=n_workers,
+        persistent_workers=args.persistent_data_loader_workers,
+    )
+
+    # 学習ステップ数を計算する
+    if args.max_train_epochs is not None:
+        args.max_train_steps = args.max_train_epochs * math.ceil(
+            len(train_dataloader) / accelerator.num_processes / args.gradient_accumulation_steps
+        )
+        accelerator.print(f"override steps. steps for {args.max_train_epochs} epochs is / 指定エポックまでのステップ数: {args.max_train_steps}")
+
+    # データセット側にも学習ステップを送信
+    train_dataset_group.set_max_train_steps(args.max_train_steps)
+
+    # lr schedulerを用意する
+    lr_scheduler = train_util.get_scheduler_fix(args, optimizer, accelerator.num_processes)
+
+    # 実験的機能：勾配も含めたfp16学習を行う　モデル全体をfp16にする
+    if args.full_fp16:
+        assert (
+            args.mixed_precision == "fp16"
+        ), "full_fp16 requires mixed precision='fp16' / full_fp16を使う場合はmixed_precision='fp16'を指定してください。"
+        accelerator.print("enable full fp16 training.")
+        unet.to(weight_dtype)
+        text_encoder1.to(weight_dtype)
+        text_encoder2.to(weight_dtype)
+    elif args.full_bf16:
+        assert (
+            args.mixed_precision == "bf16"
+        ), "full_bf16 requires mixed precision='bf16' / full_bf16を使う場合はmixed_precision='bf16'を指定してください。"
+        accelerator.print("enable full bf16 training.")
+        unet.to(weight_dtype)
+        text_encoder1.to(weight_dtype)
+        text_encoder2.to(weight_dtype)
+
+    # acceleratorがなんかよろしくやってくれるらしい
+    if args.train_text_encoder:
+        unet, text_encoder1, text_encoder2, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+            unet, text_encoder1, text_encoder2, optimizer, train_dataloader, lr_scheduler
+        )
+
+        # transform DDP after prepare
+        text_encoder1, text_encoder2, unet = train_util.transform_models_if_DDP([text_encoder1, text_encoder2, unet])
+    else:
+        unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(unet, optimizer, train_dataloader, lr_scheduler)
+        (unet,) = train_util.transform_models_if_DDP([unet])
+        text_encoder1.to(weight_dtype)
+        text_encoder2.to(weight_dtype)
+
+    # TextEncoderの出力をキャッシュするときにはCPUへ移動する
+    if args.cache_text_encoder_outputs:
+        # move Text Encoders for sampling images. Text Encoder doesn't work on CPU with fp16
+        text_encoder1.to("cpu", dtype=torch.float32)
+        text_encoder2.to("cpu", dtype=torch.float32)
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    else:
+        # make sure Text Encoders are on GPU
+        text_encoder1.to(accelerator.device)
+        text_encoder2.to(accelerator.device)
+
+    # 実験的機能：勾配も含めたfp16学習を行う　PyTorchにパッチを当ててfp16でのgrad scaleを有効にする
+    if args.full_fp16:
+        train_util.patch_accelerator_for_fp16_training(accelerator)
+
+    # resumeする
+    train_util.resume_from_local_or_hf_if_specified(accelerator, args)
+
+    # epoch数を計算する
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+    if (args.save_n_epoch_ratio is not None) and (args.save_n_epoch_ratio > 0):
+        args.save_every_n_epochs = math.floor(num_train_epochs / args.save_n_epoch_ratio) or 1
+
+    # 学習する
+    total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
+    accelerator.print("running training / 学習開始")
+    accelerator.print(f"  num examples / サンプル数: {train_dataset_group.num_train_images}")
+    accelerator.print(f"  num batches per epoch / 1epochのバッチ数: {len(train_dataloader)}")
+    accelerator.print(f"  num epochs / epoch数: {num_train_epochs}")
+    accelerator.print(f"  batch size per device / バッチサイズ: {args.train_batch_size}")
+    accelerator.print(
+        f"  total train batch size (with parallel & distributed & accumulation) / 総バッチサイズ（並列学習、勾配合計含む）: {total_batch_size}"
+    )
+    accelerator.print(f"  gradient accumulation steps / 勾配を合計するステップ数 = {args.gradient_accumulation_steps}")
+    accelerator.print(f"  total optimization steps / 学習ステップ数: {args.max_train_steps}")
+
+    progress_bar = tqdm(range(args.max_train_steps), smoothing=0, disable=not accelerator.is_local_main_process, desc="steps")
+    global_step = 0
+
+    noise_scheduler = DDPMScheduler(
+        beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, clip_sample=False
+    )
+    prepare_scheduler_for_custom_training(noise_scheduler, accelerator.device)
+
+    if accelerator.is_main_process:
+        accelerator.init_trackers("finetuning" if args.log_tracker_name is None else args.log_tracker_name)
+
+    for epoch in range(num_train_epochs):
+        accelerator.print(f"\nepoch {epoch+1}/{num_train_epochs}")
+        current_epoch.value = epoch + 1
+
+        for m in training_models:
+            m.train()
+
+        loss_total = 0
+        for step, batch in enumerate(train_dataloader):
+            current_step.value = global_step
+            # with accelerator.accumulate(training_models[0]):  # 複数モデルに対応していない模様だがとりあえずこうしておく
+            if True:
+                if "latents" in batch and batch["latents"] is not None:
+                    latents = batch["latents"].to(accelerator.device).to(dtype=weight_dtype)
+                else:
+                    with torch.no_grad():
+                        # latentに変換
+                        latents = vae.encode(batch["images"].to(vae_dtype)).latent_dist.sample().to(weight_dtype)
+
+                        # NaNが含まれていれば警告を表示し0に置き換える
+                        if torch.any(torch.isnan(latents)):
+                            accelerator.print("NaN found in latents, replacing with zeros")
+                            latents = torch.where(torch.isnan(latents), torch.zeros_like(latents), latents)
+                latents = latents * sdxl_model_util.VAE_SCALE_FACTOR
+                b_size = latents.shape[0]
+
+                input_ids1 = batch["input_ids"]
+                input_ids2 = batch["input_ids2"]
+                if not args.cache_text_encoder_outputs:
+                    with torch.set_grad_enabled(args.train_text_encoder):
+                        # Get the text embedding for conditioning
+                        # TODO support weighted captions
+                        # if args.weighted_captions:
+                        #     encoder_hidden_states = get_weighted_text_embeddings(
+                        #         tokenizer,
+                        #         text_encoder,
+                        #         batch["captions"],
+                        #         accelerator.device,
+                        #         args.max_token_length // 75 if args.max_token_length else 1,
+                        #         clip_skip=args.clip_skip,
+                        #     )
+                        # else:
+                        input_ids1 = input_ids1.to(accelerator.device)
+                        input_ids2 = input_ids2.to(accelerator.device)
+                        encoder_hidden_states1, encoder_hidden_states2, pool2 = sdxl_train_util.get_hidden_states(
+                            args,
+                            input_ids1,
+                            input_ids2,
+                            tokenizer1,
+                            tokenizer2,
+                            text_encoder1,
+                            text_encoder2,
+                            None if not args.full_fp16 else weight_dtype,
+                        )
+                else:
+                    encoder_hidden_states1 = []
+                    encoder_hidden_states2 = []
+                    pool2 = []
+                    for input_id1, input_id2 in zip(input_ids1, input_ids2):
+                        input_id1_cache_key = tuple(input_id1.squeeze(0).flatten().tolist())
+                        input_id2_cache_key = tuple(input_id2.squeeze(0).flatten().tolist())
+                        encoder_hidden_states1.append(text_encoder1_cache[input_id1_cache_key])
+                        hidden_states2, p2 = text_encoder2_cache[input_id2_cache_key]
+                        encoder_hidden_states2.append(hidden_states2)
+                        pool2.append(p2)
+                    encoder_hidden_states1 = torch.stack(encoder_hidden_states1).to(accelerator.device).to(weight_dtype)
+                    encoder_hidden_states2 = torch.stack(encoder_hidden_states2).to(accelerator.device).to(weight_dtype)
+                    pool2 = torch.stack(pool2).to(accelerator.device).to(weight_dtype)
+
+                # get size embeddings
+                orig_size = batch["original_sizes_hw"]
+                crop_size = batch["crop_top_lefts"]
+                target_size = batch["target_sizes_hw"]
+                embs = sdxl_train_util.get_size_embeddings(orig_size, crop_size, target_size, accelerator.device).to(weight_dtype)
+
+                # concat embeddings
+                vector_embedding = torch.cat([pool2, embs], dim=1).to(weight_dtype)
+                text_embedding = torch.cat([encoder_hidden_states1, encoder_hidden_states2], dim=2).to(weight_dtype)
+
+                # Sample noise, sample a random timestep for each image, and add noise to the latents,
+                # with noise offset and/or multires noise if specified
+                noise, noisy_latents, timesteps = train_util.get_noise_noisy_latents_and_timesteps(args, noise_scheduler, latents)
+
+                noisy_latents = noisy_latents.to(weight_dtype)  # TODO check why noisy_latents is not weight_dtype
+
+                # Predict the noise residual
+                with accelerator.autocast():
+                    noise_pred = unet(noisy_latents, timesteps, text_embedding, vector_embedding)
+
+                target = noise
+
+                if args.min_snr_gamma:
+                    # do not mean over batch dimension for snr weight or scale v-pred loss
+                    loss = torch.nn.functional.mse_loss(noise_pred.float(), target.float(), reduction="none")
+                    loss = loss.mean([1, 2, 3])
+
+                    if args.min_snr_gamma:
+                        loss = apply_snr_weight(loss, timesteps, noise_scheduler, args.min_snr_gamma)
+
+                    loss = loss.mean()  # mean over batch dimension
+                else:
+                    loss = torch.nn.functional.mse_loss(noise_pred.float(), target.float(), reduction="mean")
+
+                accelerator.backward(loss)
+                if accelerator.sync_gradients and args.max_grad_norm != 0.0:
+                    params_to_clip = []
+                    for m in training_models:
+                        params_to_clip.extend(m.parameters())
+                    accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
+
+                optimizer.step()
+                lr_scheduler.step()
+                optimizer.zero_grad(set_to_none=True)
+
+            # Checks if the accelerator has performed an optimization step behind the scenes
+            if accelerator.sync_gradients:
+                progress_bar.update(1)
+                global_step += 1
+
+                sdxl_train_util.sample_images(
+                    accelerator,
+                    args,
+                    None,
+                    global_step,
+                    accelerator.device,
+                    vae,
+                    [tokenizer1, tokenizer2],
+                    [text_encoder1, text_encoder2],
+                    unet,
+                )
+
+                # 指定ステップごとにモデルを保存
+                if args.save_every_n_steps is not None and global_step % args.save_every_n_steps == 0:
+                    accelerator.wait_for_everyone()
+                    if accelerator.is_main_process:
+                        src_path = src_stable_diffusion_ckpt if save_stable_diffusion_format else src_diffusers_model_path
+                        sdxl_train_util.save_sd_model_on_epoch_end_or_stepwise(
+                            args,
+                            False,
+                            accelerator,
+                            src_path,
+                            save_stable_diffusion_format,
+                            use_safetensors,
+                            save_dtype,
+                            epoch,
+                            num_train_epochs,
+                            global_step,
+                            accelerator.unwrap_model(text_encoder1),
+                            accelerator.unwrap_model(text_encoder2),
+                            accelerator.unwrap_model(unet),
+                            vae,
+                            logit_scale,
+                            ckpt_info,
+                        )
+
+            current_loss = loss.detach().item()  # 平均なのでbatch sizeは関係ないはず
+            if args.logging_dir is not None:
+                logs = {"loss": current_loss, "lr": float(lr_scheduler.get_last_lr()[0])}
+                if (
+                    args.optimizer_type.lower().startswith("DAdapt".lower()) or args.optimizer_type.lower() == "Prodigy"
+                ):  # tracking d*lr value
+                    logs["lr/d*lr"] = (
+                        lr_scheduler.optimizers[0].param_groups[0]["d"] * lr_scheduler.optimizers[0].param_groups[0]["lr"]
+                    )
+                accelerator.log(logs, step=global_step)
+
+            # TODO moving averageにする
+            loss_total += current_loss
+            avr_loss = loss_total / (step + 1)
+            logs = {"loss": avr_loss}  # , "lr": lr_scheduler.get_last_lr()[0]}
+            progress_bar.set_postfix(**logs)
+
+            if global_step >= args.max_train_steps:
+                break
+
+        if args.logging_dir is not None:
+            logs = {"loss/epoch": loss_total / len(train_dataloader)}
+            accelerator.log(logs, step=epoch + 1)
+
+        accelerator.wait_for_everyone()
+
+        if args.save_every_n_epochs is not None:
+            if accelerator.is_main_process:
+                src_path = src_stable_diffusion_ckpt if save_stable_diffusion_format else src_diffusers_model_path
+                sdxl_train_util.save_sd_model_on_epoch_end_or_stepwise(
+                    args,
+                    True,
+                    accelerator,
+                    src_path,
+                    save_stable_diffusion_format,
+                    use_safetensors,
+                    save_dtype,
+                    epoch,
+                    num_train_epochs,
+                    global_step,
+                    accelerator.unwrap_model(text_encoder1),
+                    accelerator.unwrap_model(text_encoder2),
+                    accelerator.unwrap_model(unet),
+                    vae,
+                    logit_scale,
+                    ckpt_info,
+                )
+
+        sdxl_train_util.sample_images(
+            accelerator,
+            args,
+            epoch + 1,
+            global_step,
+            accelerator.device,
+            vae,
+            [tokenizer1, tokenizer2],
+            [text_encoder1, text_encoder2],
+            unet,
+        )
+
+    is_main_process = accelerator.is_main_process
+    # if is_main_process:
+    unet = accelerator.unwrap_model(unet)
+    text_encoder1 = accelerator.unwrap_model(text_encoder1)
+    text_encoder2 = accelerator.unwrap_model(text_encoder2)
+
+    accelerator.end_training()
+
+    if args.save_state:  # and is_main_process:
+        train_util.save_state_on_train_end(args, accelerator)
+
+    del accelerator  # この後メモリを使うのでこれは消す
+
+    if is_main_process:
+        src_path = src_stable_diffusion_ckpt if save_stable_diffusion_format else src_diffusers_model_path
+        sdxl_train_util.save_sd_model_on_train_end(
+            args,
+            src_path,
+            save_stable_diffusion_format,
+            use_safetensors,
+            save_dtype,
+            epoch,
+            global_step,
+            text_encoder1,
+            text_encoder2,
+            unet,
+            vae,
+            logit_scale,
+            ckpt_info,
+        )
+        print("model saved.")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    train_util.add_sd_models_arguments(parser)
+    train_util.add_dataset_arguments(parser, True, True, True)
+    train_util.add_training_arguments(parser, False)
+    train_util.add_sd_saving_arguments(parser)
+    train_util.add_optimizer_arguments(parser)
+    config_util.add_config_arguments(parser)
+    custom_train_functions.add_custom_train_arguments(parser)
+    sdxl_train_util.add_sdxl_training_arguments(parser)
+
+    parser.add_argument("--diffusers_xformers", action="store_true", help="use xformers by diffusers / Diffusersでxformersを使用する")
+    parser.add_argument("--train_text_encoder", action="store_true", help="train text encoder / text encoderも学習する")
+    parser.add_argument(
+        "--no_half_vae",
+        action="store_true",
+        help="do not use fp16/bf16 VAE in mixed precision (use float VAE) / mixed precisionでも fp16/bf16 VAEを使わずfloat VAEを使う",
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    args = train_util.read_config_from_file(args, parser)
+
+    train(args)
diff --git a/sdxl_train_network.py b/sdxl_train_network.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c3c0cc5b89ae8b69d39472f645ea266769cd891
--- /dev/null
+++ b/sdxl_train_network.py
@@ -0,0 +1,167 @@
+import argparse
+import torch
+from library import sdxl_model_util, sdxl_train_util, train_util
+import train_network
+
+
+class SdxlNetworkTrainer(train_network.NetworkTrainer):
+    def __init__(self):
+        super().__init__()
+        self.vae_scale_factor = sdxl_model_util.VAE_SCALE_FACTOR
+
+    def assert_extra_args(self, args, train_dataset_group):
+        super().assert_extra_args(args, train_dataset_group)
+        sdxl_train_util.verify_sdxl_training_args(args)
+
+        if args.cache_text_encoder_outputs:
+            assert (
+                train_dataset_group.is_text_encoder_output_cacheable()
+            ), "when caching Text Encoder output, either caption_dropout_rate, shuffle_caption, token_warmup_step or caption_tag_dropout_rate cannot be used / Text Encoderの出力をキャッシュするときはcaption_dropout_rate, shuffle_caption, token_warmup_step, caption_tag_dropout_rateは使えません"
+
+        assert (
+            args.network_train_unet_only or not args.cache_text_encoder_outputs
+        ), "network for Text Encoder cannot be trained with caching Text Encoder outputs / Text Encoderの出力をキャッシュしながらText Encoderのネットワークを学習することはできません"
+
+    def load_target_model(self, args, weight_dtype, accelerator):
+        (
+            load_stable_diffusion_format,
+            text_encoder1,
+            text_encoder2,
+            vae,
+            unet,
+            logit_scale,
+            ckpt_info,
+        ) = sdxl_train_util.load_target_model(args, accelerator, sdxl_model_util.MODEL_VERSION_SDXL_BASE_V0_9, weight_dtype)
+
+        self.load_stable_diffusion_format = load_stable_diffusion_format
+        self.logit_scale = logit_scale
+        self.ckpt_info = ckpt_info
+
+        return sdxl_model_util.MODEL_VERSION_SDXL_BASE_V0_9, [text_encoder1, text_encoder2], vae, unet
+
+    def load_tokenizer(self, args):
+        tokenizer = sdxl_train_util.load_tokenizers(args)
+        return tokenizer
+
+    def is_text_encoder_outputs_cached(self, args):
+        return args.cache_text_encoder_outputs
+
+    def cache_text_encoder_outputs_if_needed(
+        self, args, accelerator, unet, vae, tokenizers, text_encoders, dataset, weight_dtype
+    ):
+        if args.cache_text_encoder_outputs:
+            if not args.lowram:
+                # メモリ消費を減らす
+                print("move vae and unet to cpu to save memory")
+                org_vae_device = vae.device
+                org_unet_device = unet.device
+                vae.to("cpu")
+                unet.to("cpu")
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
+            text_encoder1_cache, text_encoder2_cache = sdxl_train_util.cache_text_encoder_outputs(
+                args, accelerator, tokenizers, text_encoders, dataset, weight_dtype
+            )
+            accelerator.wait_for_everyone()
+            text_encoders[0].to("cpu", dtype=torch.float32) # Text Encoder doesn't work with fp16 on CPU
+            text_encoders[1].to("cpu", dtype=torch.float32)
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+
+            self.text_encoder1_cache = text_encoder1_cache
+            self.text_encoder2_cache = text_encoder2_cache
+
+            if not args.lowram:
+                print("move vae and unet back to original device")
+                vae.to(org_vae_device)
+                unet.to(org_unet_device)
+        else:
+            self.text_encoder1_cache = None
+            self.text_encoder2_cache = None
+
+            # Text Encoderから毎回出力を取得するので、GPUに乗せておく
+            text_encoders[0].to(accelerator.device)
+            text_encoders[1].to(accelerator.device)
+
+    def get_text_cond(self, args, accelerator, batch, tokenizers, text_encoders, weight_dtype):
+        input_ids1 = batch["input_ids"]
+        input_ids2 = batch["input_ids2"]
+        if not args.cache_text_encoder_outputs:
+            with torch.enable_grad():
+                # Get the text embedding for conditioning
+                # TODO support weighted captions
+                # if args.weighted_captions:
+                #     encoder_hidden_states = get_weighted_text_embeddings(
+                #         tokenizer,
+                #         text_encoder,
+                #         batch["captions"],
+                #         accelerator.device,
+                #         args.max_token_length // 75 if args.max_token_length else 1,
+                #         clip_skip=args.clip_skip,
+                #     )
+                # else:
+                input_ids1 = input_ids1.to(accelerator.device)
+                input_ids2 = input_ids2.to(accelerator.device)
+                encoder_hidden_states1, encoder_hidden_states2, pool2 = sdxl_train_util.get_hidden_states(
+                    args,
+                    input_ids1,
+                    input_ids2,
+                    tokenizers[0],
+                    tokenizers[1],
+                    text_encoders[0],
+                    text_encoders[1],
+                    None if not args.full_fp16 else weight_dtype,
+                )
+        else:
+            encoder_hidden_states1 = []
+            encoder_hidden_states2 = []
+            pool2 = []
+            for input_id1, input_id2 in zip(input_ids1, input_ids2):
+                input_id1_cache_key = tuple(input_id1.flatten().tolist())
+                input_id2_cache_key = tuple(input_id2.flatten().tolist())
+                encoder_hidden_states1.append(self.text_encoder1_cache[input_id1_cache_key])
+                hidden_states2, p2 = self.text_encoder2_cache[input_id2_cache_key]
+                encoder_hidden_states2.append(hidden_states2)
+                pool2.append(p2)
+            encoder_hidden_states1 = torch.stack(encoder_hidden_states1).to(accelerator.device).to(weight_dtype)
+            encoder_hidden_states2 = torch.stack(encoder_hidden_states2).to(accelerator.device).to(weight_dtype)
+            pool2 = torch.stack(pool2).to(accelerator.device).to(weight_dtype)
+
+        return encoder_hidden_states1, encoder_hidden_states2, pool2
+
+    def call_unet(self, args, accelerator, unet, noisy_latents, timesteps, text_conds, batch, weight_dtype):
+        noisy_latents = noisy_latents.to(weight_dtype)  # TODO check why noisy_latents is not weight_dtype
+
+        # get size embeddings
+        orig_size = batch["original_sizes_hw"]
+        crop_size = batch["crop_top_lefts"]
+        target_size = batch["target_sizes_hw"]
+        embs = sdxl_train_util.get_size_embeddings(orig_size, crop_size, target_size, accelerator.device).to(weight_dtype)
+
+        # concat embeddings
+        encoder_hidden_states1, encoder_hidden_states2, pool2 = text_conds
+        vector_embedding = torch.cat([pool2, embs], dim=1).to(weight_dtype)
+        text_embedding = torch.cat([encoder_hidden_states1, encoder_hidden_states2], dim=2).to(weight_dtype)
+
+        noise_pred = unet(noisy_latents, timesteps, text_embedding, vector_embedding)
+        return noise_pred
+
+    def sample_images(self, accelerator, args, epoch, global_step, device, vae, tokenizer, text_encoder, unet):
+        sdxl_train_util.sample_images(accelerator, args, epoch, global_step, device, vae, tokenizer, text_encoder, unet)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = train_network.setup_parser()
+    sdxl_train_util.add_sdxl_training_arguments(parser)
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    args = train_util.read_config_from_file(args, parser)
+
+    trainer = SdxlNetworkTrainer()
+    trainer.train(args)
diff --git a/sdxl_train_textual_inversion.py b/sdxl_train_textual_inversion.py
new file mode 100644
index 0000000000000000000000000000000000000000..9df370927333068abd19ae4209001c24cc6b86fc
--- /dev/null
+++ b/sdxl_train_textual_inversion.py
@@ -0,0 +1,142 @@
+import argparse
+import os
+
+import regex
+import torch
+import open_clip
+from library import sdxl_model_util, sdxl_train_util, train_util
+
+import train_textual_inversion
+
+
+class SdxlTextualInversionTrainer(train_textual_inversion.TextualInversionTrainer):
+    def __init__(self):
+        super().__init__()
+        self.vae_scale_factor = sdxl_model_util.VAE_SCALE_FACTOR
+
+    def assert_extra_args(self, args, train_dataset_group):
+        super().assert_extra_args(args, train_dataset_group)
+        sdxl_train_util.verify_sdxl_training_args(args)
+
+    def load_target_model(self, args, weight_dtype, accelerator):
+        (
+            load_stable_diffusion_format,
+            text_encoder1,
+            text_encoder2,
+            vae,
+            unet,
+            logit_scale,
+            ckpt_info,
+        ) = sdxl_train_util.load_target_model(args, accelerator, sdxl_model_util.MODEL_VERSION_SDXL_BASE_V0_9, weight_dtype)
+
+        self.load_stable_diffusion_format = load_stable_diffusion_format
+        self.logit_scale = logit_scale
+        self.ckpt_info = ckpt_info
+
+        return sdxl_model_util.MODEL_VERSION_SDXL_BASE_V0_9, [text_encoder1, text_encoder2], vae, unet
+
+    def load_tokenizer(self, args):
+        tokenizer = sdxl_train_util.load_tokenizers(args)
+        return tokenizer
+
+    def assert_token_string(self, token_string, tokenizers):
+        # tokenizer 1 is seems to be ok
+
+        # count words for token string: regular expression from open_clip
+        pat = regex.compile(r"""'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", regex.IGNORECASE)
+        words = regex.findall(pat, token_string)
+        word_count = len(words)
+        assert word_count == 1, (
+            f"token string {token_string} contain {word_count} words, please don't use digits, punctuation, or special characters"
+            + f" / トークン文字列 {token_string} には{word_count}個の単語が含まれています。数字、句読点、特殊文字は使用しないでください"
+        )
+
+    def get_text_cond(self, args, accelerator, batch, tokenizers, text_encoders, weight_dtype):
+        input_ids1 = batch["input_ids"]
+        input_ids2 = batch["input_ids2"]
+        with torch.enable_grad():
+            input_ids1 = input_ids1.to(accelerator.device)
+            input_ids2 = input_ids2.to(accelerator.device)
+            encoder_hidden_states1, encoder_hidden_states2, pool2 = sdxl_train_util.get_hidden_states(
+                args,
+                input_ids1,
+                input_ids2,
+                tokenizers[0],
+                tokenizers[1],
+                text_encoders[0],
+                text_encoders[1],
+                None if not args.full_fp16 else weight_dtype,
+            )
+        return encoder_hidden_states1, encoder_hidden_states2, pool2
+
+    def call_unet(self, args, accelerator, unet, noisy_latents, timesteps, text_conds, batch, weight_dtype):
+        noisy_latents = noisy_latents.to(weight_dtype)  # TODO check why noisy_latents is not weight_dtype
+
+        # get size embeddings
+        orig_size = batch["original_sizes_hw"]
+        crop_size = batch["crop_top_lefts"]
+        target_size = batch["target_sizes_hw"]
+        embs = sdxl_train_util.get_size_embeddings(orig_size, crop_size, target_size, accelerator.device).to(weight_dtype)
+
+        # concat embeddings
+        encoder_hidden_states1, encoder_hidden_states2, pool2 = text_conds
+        vector_embedding = torch.cat([pool2, embs], dim=1).to(weight_dtype)
+        text_embedding = torch.cat([encoder_hidden_states1, encoder_hidden_states2], dim=2).to(weight_dtype)
+
+        noise_pred = unet(noisy_latents, timesteps, text_embedding, vector_embedding)
+        return noise_pred
+
+    def sample_images(self, accelerator, args, epoch, global_step, device, vae, tokenizer, text_encoder, unet, prompt_replacement):
+        sdxl_train_util.sample_images(
+            accelerator, args, epoch, global_step, device, vae, tokenizer, text_encoder, unet, prompt_replacement
+        )
+
+    def save_weights(self, file, updated_embs, save_dtype):
+        state_dict = {"clip_l": updated_embs[0], "clip_g": updated_embs[1]}
+
+        if save_dtype is not None:
+            for key in list(state_dict.keys()):
+                v = state_dict[key]
+                v = v.detach().clone().to("cpu").to(save_dtype)
+                state_dict[key] = v
+
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import save_file
+
+            save_file(state_dict, file)
+        else:
+            torch.save(state_dict, file)
+
+    def load_weights(self, file):
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import load_file
+
+            data = load_file(file)
+        else:
+            data = torch.load(file, map_location="cpu")
+
+        emb_l = data.get("clib_l", None)  # ViT-L text encoder 1
+        emb_g = data.get("clib_g", None)  # BiG-G text encoder 2
+
+        assert (
+            emb_l is not None or emb_g is not None
+        ), f"weight file does not contains weights for text encoder 1 or 2 / 重みファイルにテキストエンコーダー1または2の重みが含まれていません: {file}"
+
+        return [emb_l, emb_g]
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = train_textual_inversion.setup_parser()
+    # don't add sdxl_train_util.add_sdxl_training_arguments(parser): because it only adds text encoder caching
+    # sdxl_train_util.add_sdxl_training_arguments(parser)
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    args = train_util.read_config_from_file(args, parser)
+
+    trainer = SdxlTextualInversionTrainer()
+    trainer.train(args)
diff --git a/setup-runpod.sh b/setup-runpod.sh
new file mode 100644
index 0000000000000000000000000000000000000000..4654248b2c043a3baf686a1bd97638832c06b921
--- /dev/null
+++ b/setup-runpod.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# This gets the directory the script is run from so pathing can work relative to the script where needed.
+SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)"
+
+# Install tk and python3.10-venv
+echo "Installing tk and python3.10-venv..."
+apt update -y && apt install -y python3-tk python3.10-venv
+
+# Install required libcudnn release 8.7.0.84-1
+echo "Installing required libcudnn release 8.7.0.84-1..."
+apt install -y libcudnn8=8.7.0.84-1+cuda11.8 libcudnn8-dev=8.7.0.84-1+cuda11.8 --allow-change-held-packages
+
+# Check if the venv folder doesn't exist
+if [ ! -d "$SCRIPT_DIR/venv" ]; then
+    echo "Creating venv..."
+    python3 -m venv "$SCRIPT_DIR/venv"
+fi
+
+# Activate the virtual environment
+echo "Activating venv..."
+source "$SCRIPT_DIR/venv/bin/activate" || exit 1
+
+# Run setup_linux.py script with platform requirements
+echo "Running setup_linux.py..."
+python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_runpod.txt --show_stdout --no_run_accelerate
+pip3 cache purge
+
+# Configure accelerate
+echo "Configuring accelerate..."
+mkdir -p "/root/.cache/huggingface/accelerate"
+cp "$SCRIPT_DIR/config_files/accelerate/runpod.yaml" "/root/.cache/huggingface/accelerate/default_config.yaml"
+
+echo "Installation completed... You can start the gui with ./gui.sh --share --headless"
+
+# Deactivate the virtual environment
+echo "Deactivating venv..."
+deactivate
\ No newline at end of file
diff --git a/setup.bat b/setup.bat
new file mode 100644
index 0000000000000000000000000000000000000000..dc68cbd34f96cd8eca48870e9507e531fa523823
--- /dev/null
+++ b/setup.bat
@@ -0,0 +1,37 @@
+@echo off
+
+set PYTHON_VER=3.10.9
+
+:: Check if Python version meets the recommended version
+python --version 2>nul | findstr /b /c:"Python %PYTHON_VER%" >nul
+if errorlevel 1 (
+    echo Warning: Python version %PYTHON_VER% is recommended.
+)
+
+IF NOT EXIST venv (
+    echo Creating venv...
+    python -m venv venv
+)
+
+:: Create the directory if it doesn't exist
+mkdir ".\logs\setup" > nul 2>&1
+
+:: Deactivate the virtual environment
+call .\venv\Scripts\deactivate.bat
+
+:: Calling external python program to check for local modules
+python .\setup\check_local_modules.py
+
+call .\venv\Scripts\activate.bat
+
+REM Check if the batch was started via double-click
+IF /i "%comspec% /c %~0 " equ "%cmdcmdline:"=%" (
+    REM echo This script was started by double clicking.
+    cmd /k python .\setup\setup_windows.py
+) ELSE (
+    REM echo This script was started from a command prompt.
+    python .\setup\setup_windows.py
+)
+
+:: Deactivate the virtual environment
+call .\venv\Scripts\deactivate.bat
\ No newline at end of file
diff --git a/setup.log b/setup.log
new file mode 100644
index 0000000000000000000000000000000000000000..9c24e7c4c36ea36d2fb44bcd00c1bca77528259d
--- /dev/null
+++ b/setup.log
@@ -0,0 +1,4144 @@
+2023-08-14 13:15:27,865 | INFO | /content/kohya_ss/kohya_gui.py | headless: True
+2023-08-14 13:15:27,873 | INFO | /content/kohya_ss/kohya_gui.py | Load CSS...
+2023-08-14 13:15:27,881 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | Starting new HTTPS connection (1): checkip.amazonaws.com:443
+2023-08-14 13:15:28,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | https://checkip.amazonaws.com:443 "GET / HTTP/1.1" 200 15
+2023-08-14 13:15:28,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | Starting new HTTPS connection (1): api.gradio.app:443
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,821 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,823 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,836 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,869 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,872 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,872 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,872 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,872 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,872 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,872 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,872 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,873 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,873 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,873 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,876 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,880 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,880 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,880 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,880 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,880 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,882 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,882 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,882 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,882 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,882 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,888 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,894 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,894 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,894 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,894 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,894 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,894 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,895 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,895 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,895 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,895 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,899 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,899 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,899 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,899 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,899 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,899 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,899 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,899 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,900 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,900 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,905 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,909 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,913 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,917 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,919 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,922 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,922 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,922 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,922 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,922 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,923 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,923 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,923 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,923 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,923 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,925 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=2,level=0,tokens=3), 2, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=4,level=0,tokens=6), 4, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=6,level=0,tokens=9), 6, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=6,level=0,tokens=9), 6, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=6,level=0,tokens=9), 6, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=6,level=0,tokens=9), 6, 26, False
+2023-08-14 13:15:28,926 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=6,level=0,tokens=9), 6, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=6,level=0,tokens=9), 6, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=6,level=0,tokens=9), 6, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=6,level=0,tokens=9), 6, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=8,level=0,tokens=12), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=0,tokens=12), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=0,tokens=12), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=0,tokens=12), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=0,tokens=12), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=2,tokens=14), 9, 26, True
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=2,tokens=14), 9, 26, True
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=2,tokens=14), 9, 26, True
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=2,tokens=14), 9, 26, True
+2023-08-14 13:15:28,927 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=8,level=2,tokens=14), 8, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=2,tokens=14), 9, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=2,tokens=14), 9, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=2,tokens=14), 9, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=2,tokens=14), 9, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=1,tokens=18), 9, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=1,tokens=18), 9, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=1,tokens=18), 9, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=2,tokens=19), 10, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=2,tokens=19), 10, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=2,tokens=19), 10, 26, True
+2023-08-14 13:15:28,928 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=9,level=2,tokens=19), 10, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=9,level=2,tokens=19), 9, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=2,tokens=19), 10, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=2,tokens=19), 10, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=2,tokens=19), 10, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=9,level=2,tokens=19), 10, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=1,tokens=23), 10, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=1,tokens=23), 10, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=1,tokens=23), 10, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=2,tokens=24), 11, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=2,tokens=24), 11, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=2,tokens=24), 11, 26, True
+2023-08-14 13:15:28,929 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=2,tokens=24), 11, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=10,level=2,tokens=24), 10, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=2,tokens=24), 11, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=2,tokens=24), 11, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=2,tokens=24), 11, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=2,tokens=24), 11, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=1,tokens=28), 11, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=1,tokens=28), 11, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=1,tokens=28), 11, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=11,level=2,tokens=29), 11, 26, False
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=1,tokens=33), 13, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=1,tokens=33), 13, 26, True
+2023-08-14 13:15:28,930 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=1,tokens=33), 13, 26, True
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=13,level=0,tokens=34), 13, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=15,level=0,tokens=37), 15, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=15,level=0,tokens=37), 15, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=23,level=0,tokens=38), 23, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=23,level=0,tokens=38), 23, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=23,level=0,tokens=38), 23, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=23,level=0,tokens=38), 23, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=23,level=0,tokens=38), 23, 26, False
+2023-08-14 13:15:28,931 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=23,level=0,tokens=38), 23, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=23,level=0,tokens=38), 23, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=23,level=0,tokens=38), 23, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:28,932 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=25,level=0,tokens=41), 25, 26, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,306 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,323 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,323 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,323 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,323 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,323 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,324 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,324 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,324 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,324 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,324 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,325 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,325 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,325 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,325 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,325 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,325 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,326 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,326 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,326 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,326 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,338 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,339 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,339 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,339 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,339 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,339 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,340 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,340 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,340 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,341 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,367 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,367 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,368 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,368 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,368 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,368 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,368 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,368 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,368 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,368 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,370 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 28, False
+2023-08-14 13:15:29,370 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 28, False
+2023-08-14 13:15:29,370 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 28, False
+2023-08-14 13:15:29,370 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 28, False
+2023-08-14 13:15:29,370 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 28, False
+2023-08-14 13:15:29,370 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 28, False
+2023-08-14 13:15:29,370 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=2,level=0,tokens=3), 2, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,371 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=4,level=0,tokens=6), 4, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=6,level=0,tokens=9), 6, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=6,level=0,tokens=9), 6, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=6,level=0,tokens=9), 6, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=6,level=0,tokens=9), 6, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=6,level=0,tokens=9), 6, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=6,level=0,tokens=9), 6, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=6,level=0,tokens=9), 6, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=6,level=0,tokens=9), 6, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=8,level=0,tokens=12), 8, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=0,tokens=12), 8, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=0,tokens=12), 8, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=0,tokens=12), 8, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=0,tokens=12), 8, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,372 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=2,tokens=14), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=2,tokens=14), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=2,tokens=14), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=2,tokens=14), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=8,level=2,tokens=14), 8, 28, False
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=2,tokens=14), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=2,tokens=14), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=2,tokens=14), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=2,tokens=14), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=1,tokens=18), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=1,tokens=18), 9, 28, True
+2023-08-14 13:15:29,373 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=1,tokens=18), 9, 28, True
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=2,tokens=19), 10, 28, True
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=2,tokens=19), 10, 28, True
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=2,tokens=19), 10, 28, True
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=9,level=2,tokens=19), 10, 28, True
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=9,level=2,tokens=19), 9, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=2,tokens=19), 10, 28, True
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=2,tokens=19), 10, 28, True
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=2,tokens=19), 10, 28, True
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=9,level=2,tokens=19), 10, 28, True
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=10,level=2,tokens=22), 10, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=2,tokens=22), 10, 28, False
+2023-08-14 13:15:29,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=2,tokens=22), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=2,tokens=22), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=2,tokens=22), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=4,tokens=24), 11, 28, True
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=4,tokens=24), 11, 28, True
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=4,tokens=24), 11, 28, True
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=4,tokens=24), 11, 28, True
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=10,level=4,tokens=24), 10, 28, False
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=4,tokens=24), 11, 28, True
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=4,tokens=24), 11, 28, True
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=4,tokens=24), 11, 28, True
+2023-08-14 13:15:29,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=4,tokens=24), 11, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=3,tokens=28), 11, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=3,tokens=28), 11, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=3,tokens=28), 11, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=4,tokens=29), 12, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=4,tokens=29), 12, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=4,tokens=29), 12, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=11,level=4,tokens=29), 12, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=11,level=4,tokens=29), 11, 28, False
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=4,tokens=29), 12, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=4,tokens=29), 12, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=4,tokens=29), 12, 28, True
+2023-08-14 13:15:29,376 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=11,level=4,tokens=29), 12, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=12,level=3,tokens=33), 12, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=12,level=3,tokens=33), 12, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=12,level=3,tokens=33), 12, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=12,level=4,tokens=34), 13, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=12,level=4,tokens=34), 13, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=12,level=4,tokens=34), 13, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=12,level=4,tokens=34), 13, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=12,level=4,tokens=34), 12, 28, False
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=12,level=4,tokens=34), 13, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=12,level=4,tokens=34), 13, 28, True
+2023-08-14 13:15:29,377 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=12,level=4,tokens=34), 13, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=12,level=4,tokens=34), 13, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=3,tokens=38), 13, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=3,tokens=38), 13, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=3,tokens=38), 13, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=4,tokens=39), 14, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=4,tokens=39), 14, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=4,tokens=39), 14, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=13,level=4,tokens=39), 14, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=13,level=4,tokens=39), 13, 28, False
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=4,tokens=39), 14, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=4,tokens=39), 14, 28, True
+2023-08-14 13:15:29,378 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=4,tokens=39), 14, 28, True
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=13,level=4,tokens=39), 14, 28, True
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=14,level=1,tokens=45), 14, 28, True
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=14,level=1,tokens=45), 14, 28, True
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=14,level=1,tokens=45), 14, 28, True
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=14,level=2,tokens=46), 14, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=16,level=1,tokens=50), 16, 28, True
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=16,level=1,tokens=50), 16, 28, True
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=16,level=1,tokens=50), 16, 28, True
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,379 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=16,level=0,tokens=51), 17, 28, True
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=16,level=0,tokens=51), 16, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=16,level=0,tokens=51), 17, 28, True
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=17,level=0,tokens=54), 17, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=17,level=0,tokens=54), 17, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=25,level=0,tokens=55), 25, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=25,level=0,tokens=55), 25, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=25,level=0,tokens=55), 25, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=25,level=0,tokens=55), 25, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=25,level=0,tokens=55), 25, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=25,level=0,tokens=55), 25, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=25,level=0,tokens=55), 25, 28, False
+2023-08-14 13:15:29,380 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=25,level=0,tokens=55), 25, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,381 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=27,level=0,tokens=58), 27, 28, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,384 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,387 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,387 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,388 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,388 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,388 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,388 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,388 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,388 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,388 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,388 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,391 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,391 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,391 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,391 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,391 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,391 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,391 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,392 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,392 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,392 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,395 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,399 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,399 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,400 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,400 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,400 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,400 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,400 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,400 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,400 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,400 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,405 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,408 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,410 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,413 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,413 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,413 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,413 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,414 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,414 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,414 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,414 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,414 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,414 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,418 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,418 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,418 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,418 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,418 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,418 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,418 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,418 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,419 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,419 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,424 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,424 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,424 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,424 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,424 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,425 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,425 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,425 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,425 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,425 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,431 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,441 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,442 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,442 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,442 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,442 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,442 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,442 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,442 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,442 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,443 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,448 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,449 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,449 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,449 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,449 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,449 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,449 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,449 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,449 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,449 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,453 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,456 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,456 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,456 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,456 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,456 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,457 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,457 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,457 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,457 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,457 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=0,level=0,tokens=0), 0, 1, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=0,level=0,tokens=0), 0, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=0,level=0,tokens=0), 0, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=0,level=0,tokens=0), 0, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=0,level=0,tokens=0), 0, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=0,level=0,tokens=0), 0, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=0,level=0,tokens=0), 0, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=0,level=0,tokens=0), 0, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=0,level=0,tokens=0), 0, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,638 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=2,level=0,tokens=3), 2, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=4,level=0,tokens=6), 4, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=6,level=0,tokens=9), 6, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=6,level=0,tokens=9), 6, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=6,level=0,tokens=9), 6, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=6,level=0,tokens=9), 6, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=6,level=0,tokens=9), 6, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=6,level=0,tokens=9), 6, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=6,level=0,tokens=9), 6, 469, False
+2023-08-14 13:15:29,639 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=6,level=0,tokens=9), 6, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=8,level=0,tokens=12), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=0,tokens=12), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=0,tokens=12), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=0,tokens=12), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=0,tokens=12), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=2,tokens=14), 9, 469, True
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=2,tokens=14), 9, 469, True
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=2,tokens=14), 9, 469, True
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=2,tokens=14), 9, 469, True
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=8,level=2,tokens=14), 8, 469, False
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=8,level=2,tokens=14), 9, 469, True
+2023-08-14 13:15:29,640 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=8,level=2,tokens=14), 9, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=8,level=2,tokens=14), 9, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=8,level=2,tokens=14), 9, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=1,tokens=18), 9, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=1,tokens=18), 9, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=1,tokens=18), 9, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=2,tokens=19), 10, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=2,tokens=19), 10, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=2,tokens=19), 10, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=9,level=2,tokens=19), 10, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=9,level=2,tokens=19), 9, 469, False
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=9,level=2,tokens=19), 10, 469, True
+2023-08-14 13:15:29,641 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=9,level=2,tokens=19), 10, 469, True
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=9,level=2,tokens=19), 10, 469, True
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=9,level=2,tokens=19), 10, 469, True
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=10,level=2,tokens=22), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=2,tokens=22), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=2,tokens=22), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=2,tokens=22), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=2,tokens=22), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=4,tokens=24), 11, 469, True
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=4,tokens=24), 11, 469, True
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=4,tokens=24), 11, 469, True
+2023-08-14 13:15:29,642 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=4,tokens=24), 11, 469, True
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=10,level=4,tokens=24), 10, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=10,level=4,tokens=24), 11, 469, True
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=10,level=4,tokens=24), 11, 469, True
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=10,level=4,tokens=24), 11, 469, True
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=10,level=4,tokens=24), 11, 469, True
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=11,level=4,tokens=27), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=4,tokens=27), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=4,tokens=27), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=4,tokens=27), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=11,level=4,tokens=27), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,643 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=6,tokens=29), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=6,tokens=29), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=6,tokens=29), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=11,level=6,tokens=29), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=11,level=6,tokens=29), 11, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=11,level=6,tokens=29), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=11,level=6,tokens=29), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=11,level=6,tokens=29), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=11,level=6,tokens=29), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=12,level=5,tokens=33), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=12,level=5,tokens=33), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=12,level=5,tokens=33), 12, 469, True
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,644 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=12,level=6,tokens=34), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=12,level=6,tokens=34), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=12,level=6,tokens=34), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=12,level=6,tokens=34), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=12,level=6,tokens=34), 12, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=12,level=6,tokens=34), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=12,level=6,tokens=34), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=12,level=6,tokens=34), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=12,level=6,tokens=34), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=5,tokens=38), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=5,tokens=38), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=5,tokens=38), 13, 469, True
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,645 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=6,tokens=39), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=6,tokens=39), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=6,tokens=39), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=13,level=6,tokens=39), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=13,level=6,tokens=39), 13, 469, False
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=13,level=6,tokens=39), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=13,level=6,tokens=39), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=13,level=6,tokens=39), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=13,level=6,tokens=39), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=14,level=3,tokens=45), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=14,level=3,tokens=45), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=14,level=3,tokens=45), 14, 469, True
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,646 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=14,level=4,tokens=46), 15, 469, True
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=14,level=4,tokens=46), 15, 469, True
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=14,level=4,tokens=46), 15, 469, True
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=14,level=4,tokens=46), 15, 469, True
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=14,level=4,tokens=46), 14, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=14,level=4,tokens=46), 15, 469, True
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=14,level=4,tokens=46), 15, 469, True
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=14,level=4,tokens=46), 15, 469, True
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=14,level=4,tokens=46), 15, 469, True
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=15,level=4,tokens=49), 15, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=15,level=4,tokens=49), 15, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=15,level=4,tokens=49), 15, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=15,level=4,tokens=49), 15, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=15,level=4,tokens=49), 15, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=15,level=6,tokens=51), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=15,level=6,tokens=51), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=15,level=6,tokens=51), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=15,level=6,tokens=51), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=15,level=6,tokens=51), 15, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=15,level=6,tokens=51), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=15,level=6,tokens=51), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=15,level=6,tokens=51), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=15,level=6,tokens=51), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=16,level=5,tokens=55), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=16,level=5,tokens=55), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=16,level=5,tokens=55), 16, 469, True
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,648 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=16,level=6,tokens=56), 17, 469, True
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=16,level=6,tokens=56), 17, 469, True
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=16,level=6,tokens=56), 17, 469, True
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=16,level=6,tokens=56), 17, 469, True
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=16,level=6,tokens=56), 16, 469, False
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=16,level=6,tokens=56), 17, 469, True
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=16,level=6,tokens=56), 17, 469, True
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=16,level=6,tokens=56), 17, 469, True
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=16,level=6,tokens=56), 17, 469, True
+2023-08-14 13:15:29,649 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=17,level=5,tokens=60), 17, 469, True
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=17,level=5,tokens=60), 17, 469, True
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=17,level=5,tokens=60), 17, 469, True
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=17,level=6,tokens=61), 18, 469, True
+2023-08-14 13:15:29,650 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=17,level=6,tokens=61), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=17,level=6,tokens=61), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=17,level=6,tokens=61), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=17,level=6,tokens=61), 17, 469, False
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=17,level=6,tokens=61), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=17,level=6,tokens=61), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=17,level=6,tokens=61), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=17,level=6,tokens=61), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=18,level=3,tokens=67), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=18,level=3,tokens=67), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=18,level=3,tokens=67), 18, 469, True
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,651 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=18,level=4,tokens=68), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=18,level=4,tokens=68), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=18,level=4,tokens=68), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=18,level=4,tokens=68), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=18,level=4,tokens=68), 18, 469, False
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=18,level=4,tokens=68), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=18,level=4,tokens=68), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=18,level=4,tokens=68), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=18,level=4,tokens=68), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=19,level=3,tokens=72), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=19,level=3,tokens=72), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=19,level=3,tokens=72), 19, 469, True
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,652 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=19,level=4,tokens=73), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=19,level=4,tokens=73), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=19,level=4,tokens=73), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=19,level=4,tokens=73), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=19,level=4,tokens=73), 19, 469, False
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=19,level=4,tokens=73), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=19,level=4,tokens=73), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=19,level=4,tokens=73), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=19,level=4,tokens=73), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=20,level=1,tokens=79), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=20,level=1,tokens=79), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=20,level=1,tokens=79), 20, 469, True
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,653 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=20,level=2,tokens=80), 21, 469, True
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=20,level=2,tokens=80), 21, 469, True
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=20,level=2,tokens=80), 21, 469, True
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=20,level=2,tokens=80), 21, 469, True
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=20,level=2,tokens=80), 20, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=20,level=2,tokens=80), 21, 469, True
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=20,level=2,tokens=80), 21, 469, True
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=20,level=2,tokens=80), 21, 469, True
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=20,level=2,tokens=80), 21, 469, True
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=21,level=2,tokens=83), 21, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=21,level=2,tokens=83), 21, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=21,level=2,tokens=83), 21, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=21,level=2,tokens=83), 21, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=21,level=2,tokens=83), 21, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,654 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=21,level=4,tokens=85), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=21,level=4,tokens=85), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=21,level=4,tokens=85), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=21,level=4,tokens=85), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=21,level=4,tokens=85), 21, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=21,level=4,tokens=85), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=21,level=4,tokens=85), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=21,level=4,tokens=85), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=21,level=4,tokens=85), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=22,level=3,tokens=89), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=22,level=3,tokens=89), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=22,level=3,tokens=89), 22, 469, True
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,655 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=22,level=4,tokens=90), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=22,level=4,tokens=90), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=22,level=4,tokens=90), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=22,level=4,tokens=90), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=22,level=4,tokens=90), 22, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=22,level=4,tokens=90), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=22,level=4,tokens=90), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=22,level=4,tokens=90), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=22,level=4,tokens=90), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=23,level=1,tokens=96), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=23,level=1,tokens=96), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=23,level=1,tokens=96), 23, 469, True
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,656 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=23,level=2,tokens=97), 24, 469, True
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=23,level=2,tokens=97), 24, 469, True
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=23,level=2,tokens=97), 24, 469, True
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=23,level=2,tokens=97), 24, 469, True
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=23,level=2,tokens=97), 23, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=23,level=2,tokens=97), 24, 469, True
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=23,level=2,tokens=97), 24, 469, True
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=23,level=2,tokens=97), 24, 469, True
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=23,level=2,tokens=97), 24, 469, True
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=24,level=2,tokens=100), 24, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=24,level=2,tokens=100), 24, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=24,level=2,tokens=100), 24, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=24,level=2,tokens=100), 24, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=24,level=2,tokens=100), 24, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,657 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=24,level=4,tokens=102), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=24,level=4,tokens=102), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=24,level=4,tokens=102), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=24,level=4,tokens=102), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=24,level=4,tokens=102), 24, 469, False
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=24,level=4,tokens=102), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=24,level=4,tokens=102), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=24,level=4,tokens=102), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=24,level=4,tokens=102), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=25,level=3,tokens=106), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=25,level=3,tokens=106), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=25,level=3,tokens=106), 25, 469, True
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,658 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=25,level=4,tokens=107), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=25,level=4,tokens=107), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=25,level=4,tokens=107), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=25,level=4,tokens=107), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=25,level=4,tokens=107), 25, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=25,level=4,tokens=107), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=25,level=4,tokens=107), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=25,level=4,tokens=107), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=25,level=4,tokens=107), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=26,level=1,tokens=113), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=26,level=1,tokens=113), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=26,level=1,tokens=113), 26, 469, True
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,659 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=26,level=2,tokens=114), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=26,level=2,tokens=114), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=26,level=2,tokens=114), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=26,level=2,tokens=114), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=26,level=2,tokens=114), 26, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=26,level=2,tokens=114), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=26,level=2,tokens=114), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=26,level=2,tokens=114), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=26,level=2,tokens=114), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=27,level=1,tokens=118), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=27,level=1,tokens=118), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=27,level=1,tokens=118), 27, 469, True
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,660 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=27,level=2,tokens=119), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=27,level=2,tokens=119), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=27,level=2,tokens=119), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=27,level=2,tokens=119), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=27,level=2,tokens=119), 27, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=27,level=2,tokens=119), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=27,level=2,tokens=119), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=27,level=2,tokens=119), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=27,level=2,tokens=119), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=28,level=1,tokens=123), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=28,level=1,tokens=123), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=28,level=1,tokens=123), 28, 469, True
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,661 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=28,level=2,tokens=124), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=28,level=2,tokens=124), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=28,level=2,tokens=124), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=28,level=2,tokens=124), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=28,level=2,tokens=124), 28, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=28,level=2,tokens=124), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=28,level=2,tokens=124), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=28,level=2,tokens=124), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=28,level=2,tokens=124), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=29,level=1,tokens=128), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=29,level=1,tokens=128), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=29,level=1,tokens=128), 29, 469, True
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,662 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=29,level=2,tokens=129), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=29,level=2,tokens=129), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=29,level=2,tokens=129), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=29,level=2,tokens=129), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=29,level=2,tokens=129), 29, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=29,level=2,tokens=129), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=29,level=2,tokens=129), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=29,level=2,tokens=129), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=29,level=2,tokens=129), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=30,level=1,tokens=133), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=30,level=1,tokens=133), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=30,level=1,tokens=133), 30, 469, True
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,663 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=30,level=2,tokens=134), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=30,level=2,tokens=134), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=30,level=2,tokens=134), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=30,level=2,tokens=134), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=30,level=2,tokens=134), 30, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=30,level=2,tokens=134), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=30,level=2,tokens=134), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=30,level=2,tokens=134), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=30,level=2,tokens=134), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=31,level=1,tokens=138), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=31,level=1,tokens=138), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=31,level=1,tokens=138), 31, 469, True
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,664 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=31,level=2,tokens=139), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=31,level=2,tokens=139), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=31,level=2,tokens=139), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=31,level=2,tokens=139), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=31,level=2,tokens=139), 31, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=31,level=2,tokens=139), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=31,level=2,tokens=139), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=31,level=2,tokens=139), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=31,level=2,tokens=139), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=32,level=1,tokens=143), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=32,level=1,tokens=143), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=32,level=1,tokens=143), 32, 469, True
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,665 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=32,level=2,tokens=144), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=32,level=2,tokens=144), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=32,level=2,tokens=144), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=32,level=2,tokens=144), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=32,level=2,tokens=144), 32, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=32,level=2,tokens=144), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=32,level=2,tokens=144), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=32,level=2,tokens=144), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=32,level=2,tokens=144), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=33,level=1,tokens=148), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=33,level=1,tokens=148), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=33,level=1,tokens=148), 33, 469, True
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,666 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=33,level=2,tokens=149), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=33,level=2,tokens=149), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=33,level=2,tokens=149), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=33,level=2,tokens=149), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=33,level=2,tokens=149), 33, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=33,level=2,tokens=149), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=33,level=2,tokens=149), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=33,level=2,tokens=149), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=33,level=2,tokens=149), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=34,level=1,tokens=153), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=34,level=1,tokens=153), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=34,level=1,tokens=153), 34, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=34,level=2,tokens=154), 35, 469, True
+2023-08-14 13:15:29,667 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=34,level=2,tokens=154), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=34,level=2,tokens=154), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=34,level=2,tokens=154), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=34,level=2,tokens=154), 34, 469, False
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=34,level=2,tokens=154), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=34,level=2,tokens=154), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=34,level=2,tokens=154), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=34,level=2,tokens=154), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=35,level=1,tokens=158), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=35,level=1,tokens=158), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=35,level=1,tokens=158), 35, 469, True
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,668 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=35,level=2,tokens=159), 35, 469, False
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=37,level=1,tokens=163), 37, 469, True
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=37,level=1,tokens=163), 37, 469, True
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=37,level=1,tokens=163), 37, 469, True
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=37,level=0,tokens=164), 37, 469, False
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=37,level=0,tokens=164), 37, 469, False
+2023-08-14 13:15:29,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=37,level=0,tokens=164), 37, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=37,level=0,tokens=164), 37, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=37,level=0,tokens=164), 37, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=37,level=0,tokens=164), 37, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=37,level=0,tokens=164), 37, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=37,level=0,tokens=164), 37, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=39,level=0,tokens=167), 39, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,670 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=41,level=0,tokens=170), 41, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=43,level=0,tokens=173), 43, 469, False
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,671 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=43,level=0,tokens=173), 44, 469, True
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=44,level=0,tokens=176), 44, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=44,level=0,tokens=176), 44, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=44,level=0,tokens=176), 44, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=44,level=0,tokens=176), 44, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=44,level=0,tokens=176), 44, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=44,level=0,tokens=176), 44, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=44,level=0,tokens=176), 44, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=44,level=0,tokens=176), 44, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=46,level=0,tokens=179), 46, 469, False
+2023-08-14 13:15:29,672 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=48,level=0,tokens=182), 48, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=50,level=0,tokens=185), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=50,level=0,tokens=185), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=50,level=0,tokens=185), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=50,level=0,tokens=185), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=50,level=0,tokens=185), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=50,level=2,tokens=187), 51, 469, True
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=50,level=2,tokens=187), 51, 469, True
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=50,level=2,tokens=187), 51, 469, True
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=50,level=2,tokens=187), 51, 469, True
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=50,level=2,tokens=187), 50, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=50,level=2,tokens=187), 51, 469, True
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=50,level=2,tokens=187), 51, 469, True
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=50,level=2,tokens=187), 51, 469, True
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=50,level=2,tokens=187), 51, 469, True
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=51,level=2,tokens=190), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=51,level=2,tokens=190), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=51,level=2,tokens=190), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=51,level=2,tokens=190), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=51,level=2,tokens=190), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,674 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=51,level=4,tokens=192), 52, 469, True
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=51,level=4,tokens=192), 52, 469, True
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=51,level=4,tokens=192), 52, 469, True
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=51,level=4,tokens=192), 52, 469, True
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=51,level=4,tokens=192), 51, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=51,level=4,tokens=192), 52, 469, True
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=51,level=4,tokens=192), 52, 469, True
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=51,level=4,tokens=192), 52, 469, True
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=51,level=4,tokens=192), 52, 469, True
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=52,level=4,tokens=195), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=52,level=4,tokens=195), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=52,level=4,tokens=195), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=52,level=4,tokens=195), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=52,level=4,tokens=195), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,675 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=52,level=6,tokens=197), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=52,level=6,tokens=197), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=52,level=6,tokens=197), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=52,level=6,tokens=197), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=52,level=6,tokens=197), 52, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=52,level=6,tokens=197), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=52,level=6,tokens=197), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=52,level=6,tokens=197), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=52,level=6,tokens=197), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=53,level=5,tokens=201), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=53,level=5,tokens=201), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=53,level=5,tokens=201), 53, 469, True
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,676 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=53,level=6,tokens=202), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=53,level=6,tokens=202), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=53,level=6,tokens=202), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=53,level=6,tokens=202), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=53,level=6,tokens=202), 53, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=53,level=6,tokens=202), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=53,level=6,tokens=202), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=53,level=6,tokens=202), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=53,level=6,tokens=202), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=54,level=5,tokens=206), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=54,level=5,tokens=206), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=54,level=5,tokens=206), 54, 469, True
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,677 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=54,level=6,tokens=207), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=54,level=6,tokens=207), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=54,level=6,tokens=207), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=54,level=6,tokens=207), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=54,level=6,tokens=207), 54, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=54,level=6,tokens=207), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=54,level=6,tokens=207), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=54,level=6,tokens=207), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=54,level=6,tokens=207), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=55,level=5,tokens=211), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=55,level=5,tokens=211), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=55,level=5,tokens=211), 55, 469, True
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,678 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=55,level=6,tokens=212), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=55,level=6,tokens=212), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=55,level=6,tokens=212), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=55,level=6,tokens=212), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=55,level=6,tokens=212), 55, 469, False
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=55,level=6,tokens=212), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=55,level=6,tokens=212), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=55,level=6,tokens=212), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=55,level=6,tokens=212), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=56,level=1,tokens=220), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=56,level=1,tokens=220), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=56,level=1,tokens=220), 56, 469, True
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,679 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=56,level=2,tokens=221), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=56,level=2,tokens=221), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=56,level=2,tokens=221), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=56,level=2,tokens=221), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=56,level=2,tokens=221), 56, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=56,level=2,tokens=221), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=56,level=2,tokens=221), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=56,level=2,tokens=221), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=56,level=2,tokens=221), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=57,level=1,tokens=225), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=57,level=1,tokens=225), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=57,level=1,tokens=225), 57, 469, True
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,680 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=57,level=2,tokens=226), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=57,level=2,tokens=226), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=57,level=2,tokens=226), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=57,level=2,tokens=226), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=57,level=2,tokens=226), 57, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=57,level=2,tokens=226), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=57,level=2,tokens=226), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=57,level=2,tokens=226), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=57,level=2,tokens=226), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=58,level=1,tokens=230), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=58,level=1,tokens=230), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=58,level=1,tokens=230), 58, 469, True
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,681 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=58,level=2,tokens=231), 59, 469, True
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=58,level=2,tokens=231), 59, 469, True
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=58,level=2,tokens=231), 59, 469, True
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=58,level=2,tokens=231), 59, 469, True
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=58,level=2,tokens=231), 58, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=58,level=2,tokens=231), 59, 469, True
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=58,level=2,tokens=231), 59, 469, True
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=58,level=2,tokens=231), 59, 469, True
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=58,level=2,tokens=231), 59, 469, True
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=59,level=2,tokens=234), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=59,level=2,tokens=234), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=59,level=2,tokens=234), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=59,level=2,tokens=234), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=59,level=2,tokens=234), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,682 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=59,level=4,tokens=236), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=59,level=4,tokens=236), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=59,level=4,tokens=236), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=59,level=4,tokens=236), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=59,level=4,tokens=236), 59, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=59,level=4,tokens=236), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=59,level=4,tokens=236), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=59,level=4,tokens=236), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=59,level=4,tokens=236), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=60,level=3,tokens=240), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=60,level=3,tokens=240), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=60,level=3,tokens=240), 60, 469, True
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,683 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=60,level=4,tokens=241), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=60,level=4,tokens=241), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=60,level=4,tokens=241), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=60,level=4,tokens=241), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=60,level=4,tokens=241), 60, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=60,level=4,tokens=241), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=60,level=4,tokens=241), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=60,level=4,tokens=241), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=60,level=4,tokens=241), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=61,level=1,tokens=247), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=61,level=1,tokens=247), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=61,level=1,tokens=247), 61, 469, True
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,684 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=61,level=2,tokens=248), 61, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=63,level=1,tokens=252), 63, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=63,level=1,tokens=252), 63, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=63,level=1,tokens=252), 63, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=63,level=2,tokens=253), 64, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=63,level=2,tokens=253), 64, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=63,level=2,tokens=253), 64, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=63,level=2,tokens=253), 64, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=63,level=2,tokens=253), 63, 469, False
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=63,level=2,tokens=253), 64, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=63,level=2,tokens=253), 64, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=63,level=2,tokens=253), 64, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=63,level=2,tokens=253), 64, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=64,level=1,tokens=257), 64, 469, True
+2023-08-14 13:15:29,685 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=64,level=1,tokens=257), 64, 469, True
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=64,level=1,tokens=257), 64, 469, True
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=64,level=2,tokens=258), 64, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=66,level=1,tokens=262), 66, 469, True
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=66,level=1,tokens=262), 66, 469, True
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=66,level=1,tokens=262), 66, 469, True
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,686 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=66,level=2,tokens=263), 67, 469, True
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=66,level=2,tokens=263), 67, 469, True
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=66,level=2,tokens=263), 67, 469, True
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=66,level=2,tokens=263), 67, 469, True
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=66,level=2,tokens=263), 66, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=66,level=2,tokens=263), 67, 469, True
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=66,level=2,tokens=263), 67, 469, True
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=66,level=2,tokens=263), 67, 469, True
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=66,level=2,tokens=263), 67, 469, True
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=67,level=2,tokens=266), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=67,level=2,tokens=266), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=67,level=2,tokens=266), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=67,level=2,tokens=266), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=67,level=2,tokens=266), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,687 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=67,level=4,tokens=268), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=67,level=4,tokens=268), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=67,level=4,tokens=268), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=67,level=4,tokens=268), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=67,level=4,tokens=268), 67, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=67,level=4,tokens=268), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=67,level=4,tokens=268), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=67,level=4,tokens=268), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=67,level=4,tokens=268), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=68,level=3,tokens=272), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=68,level=3,tokens=272), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=68,level=3,tokens=272), 68, 469, True
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,688 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=68,level=4,tokens=273), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=68,level=4,tokens=273), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=68,level=4,tokens=273), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=68,level=4,tokens=273), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=68,level=4,tokens=273), 68, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=68,level=4,tokens=273), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=68,level=4,tokens=273), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=68,level=4,tokens=273), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=68,level=4,tokens=273), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=69,level=3,tokens=277), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=69,level=3,tokens=277), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=69,level=3,tokens=277), 69, 469, True
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,689 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=69,level=4,tokens=278), 70, 469, True
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=69,level=4,tokens=278), 70, 469, True
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=69,level=4,tokens=278), 70, 469, True
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=69,level=4,tokens=278), 70, 469, True
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=69,level=4,tokens=278), 69, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=69,level=4,tokens=278), 70, 469, True
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=69,level=4,tokens=278), 70, 469, True
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=69,level=4,tokens=278), 70, 469, True
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=69,level=4,tokens=278), 70, 469, True
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=70,level=4,tokens=281), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=70,level=4,tokens=281), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=70,level=4,tokens=281), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=70,level=4,tokens=281), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=70,level=4,tokens=281), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,690 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=70,level=6,tokens=283), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=70,level=6,tokens=283), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=70,level=6,tokens=283), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=70,level=6,tokens=283), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=70,level=6,tokens=283), 70, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=70,level=6,tokens=283), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=70,level=6,tokens=283), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=70,level=6,tokens=283), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=70,level=6,tokens=283), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=71,level=5,tokens=287), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=71,level=5,tokens=287), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=71,level=5,tokens=287), 71, 469, True
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,691 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=71,level=6,tokens=288), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=71,level=6,tokens=288), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=71,level=6,tokens=288), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=71,level=6,tokens=288), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=71,level=6,tokens=288), 71, 469, False
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=71,level=6,tokens=288), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=71,level=6,tokens=288), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=71,level=6,tokens=288), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=71,level=6,tokens=288), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=72,level=3,tokens=294), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=72,level=3,tokens=294), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=72,level=3,tokens=294), 72, 469, True
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,692 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=72,level=4,tokens=295), 72, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=74,level=1,tokens=301), 74, 469, True
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=74,level=1,tokens=301), 74, 469, True
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=74,level=1,tokens=301), 74, 469, True
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=74,level=2,tokens=302), 75, 469, True
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=74,level=2,tokens=302), 75, 469, True
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=74,level=2,tokens=302), 75, 469, True
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=74,level=2,tokens=302), 75, 469, True
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=74,level=2,tokens=302), 74, 469, False
+2023-08-14 13:15:29,693 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=74,level=2,tokens=302), 75, 469, True
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=74,level=2,tokens=302), 75, 469, True
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=74,level=2,tokens=302), 75, 469, True
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=74,level=2,tokens=302), 75, 469, True
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=75,level=2,tokens=305), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=75,level=2,tokens=305), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=75,level=2,tokens=305), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=75,level=2,tokens=305), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=75,level=2,tokens=305), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=75,level=4,tokens=307), 75, 469, False
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=77,level=1,tokens=313), 77, 469, True
+2023-08-14 13:15:29,694 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=77,level=1,tokens=313), 77, 469, True
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=77,level=1,tokens=313), 77, 469, True
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=77,level=0,tokens=314), 77, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=79,level=0,tokens=317), 79, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=79,level=0,tokens=317), 79, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=79,level=0,tokens=317), 79, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=79,level=0,tokens=317), 79, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=79,level=0,tokens=317), 79, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=79,level=0,tokens=317), 79, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=79,level=0,tokens=317), 79, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=79,level=0,tokens=317), 79, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=81,level=0,tokens=320), 81, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=81,level=0,tokens=320), 81, 469, False
+2023-08-14 13:15:29,695 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=81,level=0,tokens=320), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=81,level=0,tokens=320), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=81,level=0,tokens=320), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=81,level=2,tokens=322), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=81,level=2,tokens=322), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=81,level=2,tokens=322), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=81,level=2,tokens=322), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=81,level=2,tokens=322), 81, 469, False
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=81,level=2,tokens=322), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=81,level=2,tokens=322), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=81,level=2,tokens=322), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=81,level=2,tokens=322), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=82,level=1,tokens=326), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=82,level=1,tokens=326), 82, 469, True
+2023-08-14 13:15:29,696 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=82,level=1,tokens=326), 82, 469, True
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=82,level=2,tokens=327), 83, 469, True
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=82,level=2,tokens=327), 83, 469, True
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=82,level=2,tokens=327), 83, 469, True
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=82,level=2,tokens=327), 83, 469, True
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=82,level=2,tokens=327), 82, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=82,level=2,tokens=327), 83, 469, True
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=82,level=2,tokens=327), 83, 469, True
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=82,level=2,tokens=327), 83, 469, True
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=82,level=2,tokens=327), 83, 469, True
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=83,level=2,tokens=330), 83, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=83,level=2,tokens=330), 83, 469, False
+2023-08-14 13:15:29,697 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=83,level=2,tokens=330), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=83,level=2,tokens=330), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=83,level=2,tokens=330), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=83,level=4,tokens=332), 84, 469, True
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=83,level=4,tokens=332), 84, 469, True
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=83,level=4,tokens=332), 84, 469, True
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=83,level=4,tokens=332), 84, 469, True
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=83,level=4,tokens=332), 83, 469, False
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=83,level=4,tokens=332), 84, 469, True
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=83,level=4,tokens=332), 84, 469, True
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=83,level=4,tokens=332), 84, 469, True
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=83,level=4,tokens=332), 84, 469, True
+2023-08-14 13:15:29,698 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=84,level=3,tokens=336), 84, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=84,level=3,tokens=336), 84, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=84,level=3,tokens=336), 84, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=84,level=4,tokens=337), 85, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=84,level=4,tokens=337), 85, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=84,level=4,tokens=337), 85, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=84,level=4,tokens=337), 85, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=84,level=4,tokens=337), 84, 469, False
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=84,level=4,tokens=337), 85, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=84,level=4,tokens=337), 85, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=84,level=4,tokens=337), 85, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=84,level=4,tokens=337), 85, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=85,level=3,tokens=341), 85, 469, True
+2023-08-14 13:15:29,699 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=85,level=3,tokens=341), 85, 469, True
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=85,level=3,tokens=341), 85, 469, True
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=85,level=4,tokens=342), 86, 469, True
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=85,level=4,tokens=342), 86, 469, True
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=85,level=4,tokens=342), 86, 469, True
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=85,level=4,tokens=342), 86, 469, True
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=85,level=4,tokens=342), 85, 469, False
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=85,level=4,tokens=342), 86, 469, True
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=85,level=4,tokens=342), 86, 469, True
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=85,level=4,tokens=342), 86, 469, True
+2023-08-14 13:15:29,700 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=85,level=4,tokens=342), 86, 469, True
+2023-08-14 13:15:29,701 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=86,level=3,tokens=346), 86, 469, True
+2023-08-14 13:15:29,701 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=86,level=3,tokens=346), 86, 469, True
+2023-08-14 13:15:29,701 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=86,level=3,tokens=346), 86, 469, True
+2023-08-14 13:15:29,701 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,701 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,701 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=86,level=4,tokens=347), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=86,level=4,tokens=347), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=86,level=4,tokens=347), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=86,level=4,tokens=347), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=86,level=4,tokens=347), 86, 469, False
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=86,level=4,tokens=347), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=86,level=4,tokens=347), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=86,level=4,tokens=347), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=86,level=4,tokens=347), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=87,level=1,tokens=353), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=87,level=1,tokens=353), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=87,level=1,tokens=353), 87, 469, True
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,702 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=87,level=2,tokens=354), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=87,level=2,tokens=354), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=87,level=2,tokens=354), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=87,level=2,tokens=354), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=87,level=2,tokens=354), 87, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=87,level=2,tokens=354), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=87,level=2,tokens=354), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=87,level=2,tokens=354), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=87,level=2,tokens=354), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=88,level=1,tokens=358), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=88,level=1,tokens=358), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=88,level=1,tokens=358), 88, 469, True
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,703 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=88,level=2,tokens=359), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=88,level=2,tokens=359), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=88,level=2,tokens=359), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=88,level=2,tokens=359), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=88,level=2,tokens=359), 88, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=88,level=2,tokens=359), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=88,level=2,tokens=359), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=88,level=2,tokens=359), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=88,level=2,tokens=359), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=89,level=1,tokens=363), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=89,level=1,tokens=363), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=89,level=1,tokens=363), 89, 469, True
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,704 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=89,level=2,tokens=364), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=89,level=2,tokens=364), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=89,level=2,tokens=364), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=89,level=2,tokens=364), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=89,level=2,tokens=364), 89, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=89,level=2,tokens=364), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=89,level=2,tokens=364), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=89,level=2,tokens=364), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=89,level=2,tokens=364), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=90,level=1,tokens=368), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=90,level=1,tokens=368), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=90,level=1,tokens=368), 90, 469, True
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,705 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=90,level=2,tokens=369), 90, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=92,level=1,tokens=373), 92, 469, True
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=92,level=1,tokens=373), 92, 469, True
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=92,level=1,tokens=373), 92, 469, True
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=92,level=0,tokens=374), 93, 469, True
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=92,level=0,tokens=374), 92, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=92,level=0,tokens=374), 93, 469, True
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=93,level=0,tokens=377), 93, 469, False
+2023-08-14 13:15:29,706 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=93,level=0,tokens=377), 93, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=101,level=0,tokens=378), 101, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=101,level=0,tokens=378), 101, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=101,level=0,tokens=378), 101, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=101,level=0,tokens=378), 101, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=101,level=0,tokens=378), 101, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=101,level=0,tokens=378), 101, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=101,level=0,tokens=378), 101, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=101,level=0,tokens=378), 101, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=103,level=0,tokens=381), 103, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=103,level=0,tokens=381), 103, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=110,level=0,tokens=382), 110, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=110,level=0,tokens=382), 110, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=110,level=0,tokens=382), 110, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=110,level=0,tokens=382), 110, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=110,level=0,tokens=382), 110, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=110,level=0,tokens=382), 110, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=110,level=0,tokens=382), 110, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=110,level=0,tokens=382), 110, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=112,level=0,tokens=385), 112, 469, False
+2023-08-14 13:15:29,707 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=112,level=0,tokens=385), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=112,level=0,tokens=385), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=112,level=0,tokens=385), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=112,level=0,tokens=385), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=112,level=2,tokens=387), 113, 469, True
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=112,level=2,tokens=387), 113, 469, True
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=112,level=2,tokens=387), 113, 469, True
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=112,level=2,tokens=387), 113, 469, True
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=112,level=2,tokens=387), 112, 469, False
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=112,level=2,tokens=387), 113, 469, True
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=112,level=2,tokens=387), 113, 469, True
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=112,level=2,tokens=387), 113, 469, True
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=112,level=2,tokens=387), 113, 469, True
+2023-08-14 13:15:29,708 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=113,level=1,tokens=391), 113, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=113,level=1,tokens=391), 113, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=113,level=1,tokens=391), 113, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=113,level=2,tokens=392), 114, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=113,level=2,tokens=392), 114, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=113,level=2,tokens=392), 114, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=113,level=2,tokens=392), 114, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=113,level=2,tokens=392), 113, 469, False
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=113,level=2,tokens=392), 114, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=113,level=2,tokens=392), 114, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=113,level=2,tokens=392), 114, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=113,level=2,tokens=392), 114, 469, True
+2023-08-14 13:15:29,709 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=114,level=1,tokens=396), 114, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=114,level=1,tokens=396), 114, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=114,level=1,tokens=396), 114, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=114,level=2,tokens=397), 115, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=114,level=2,tokens=397), 115, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=114,level=2,tokens=397), 115, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=114,level=2,tokens=397), 115, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=114,level=2,tokens=397), 114, 469, False
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=114,level=2,tokens=397), 115, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=114,level=2,tokens=397), 115, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=114,level=2,tokens=397), 115, 469, True
+2023-08-14 13:15:29,710 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=114,level=2,tokens=397), 115, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=115,level=1,tokens=401), 115, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=115,level=1,tokens=401), 115, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=115,level=1,tokens=401), 115, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=115,level=2,tokens=402), 116, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=115,level=2,tokens=402), 116, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=115,level=2,tokens=402), 116, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=115,level=2,tokens=402), 116, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=115,level=2,tokens=402), 115, 469, False
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=115,level=2,tokens=402), 116, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=115,level=2,tokens=402), 116, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=115,level=2,tokens=402), 116, 469, True
+2023-08-14 13:15:29,711 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=115,level=2,tokens=402), 116, 469, True
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=116,level=1,tokens=406), 116, 469, True
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=116,level=1,tokens=406), 116, 469, True
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=116,level=1,tokens=406), 116, 469, True
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=116,level=2,tokens=407), 116, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=118,level=1,tokens=411), 118, 469, True
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=118,level=1,tokens=411), 118, 469, True
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=118,level=1,tokens=411), 118, 469, True
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=118,level=0,tokens=412), 118, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=118,level=0,tokens=412), 118, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=118,level=0,tokens=412), 118, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=118,level=0,tokens=412), 118, 469, False
+2023-08-14 13:15:29,712 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=118,level=0,tokens=412), 118, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=118,level=0,tokens=412), 118, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=118,level=0,tokens=412), 118, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=118,level=0,tokens=412), 118, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=120,level=0,tokens=415), 120, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,713 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=122,level=0,tokens=418), 122, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=122,level=0,tokens=418), 123, 469, True
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,714 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=125,level=0,tokens=421), 125, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=127,level=0,tokens=424), 127, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,715 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=129,level=0,tokens=427), 129, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=131,level=0,tokens=430), 131, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=131,level=0,tokens=430), 131, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=131,level=0,tokens=430), 131, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=131,level=0,tokens=430), 131, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=131,level=0,tokens=430), 131, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=131,level=0,tokens=430), 131, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=131,level=0,tokens=430), 131, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=131,level=0,tokens=430), 131, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=133,level=0,tokens=433), 133, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=133,level=0,tokens=433), 133, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=133,level=0,tokens=433), 133, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=133,level=0,tokens=433), 133, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=133,level=0,tokens=433), 133, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=133,level=0,tokens=433), 133, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=133,level=0,tokens=433), 133, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=133,level=0,tokens=433), 133, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=135,level=0,tokens=436), 135, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=135,level=0,tokens=436), 135, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=135,level=0,tokens=436), 135, 469, False
+2023-08-14 13:15:29,716 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=135,level=0,tokens=436), 135, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=135,level=0,tokens=436), 135, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=135,level=0,tokens=436), 135, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=135,level=0,tokens=436), 135, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=135,level=0,tokens=436), 135, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=137,level=0,tokens=439), 137, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=139,level=0,tokens=442), 139, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=139,level=0,tokens=442), 139, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=139,level=0,tokens=442), 139, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=139,level=0,tokens=442), 139, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=139,level=0,tokens=442), 139, 469, False
+2023-08-14 13:15:29,717 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=139,level=2,tokens=444), 140, 469, True
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=139,level=2,tokens=444), 140, 469, True
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=139,level=2,tokens=444), 140, 469, True
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=139,level=2,tokens=444), 140, 469, True
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=139,level=2,tokens=444), 139, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=139,level=2,tokens=444), 140, 469, True
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=139,level=2,tokens=444), 140, 469, True
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=139,level=2,tokens=444), 140, 469, True
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=139,level=2,tokens=444), 140, 469, True
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=140,level=2,tokens=447), 140, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=140,level=2,tokens=447), 140, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=140,level=2,tokens=447), 140, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=140,level=2,tokens=447), 140, 469, False
+2023-08-14 13:15:29,718 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=140,level=2,tokens=447), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=140,level=4,tokens=449), 140, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=142,level=1,tokens=455), 142, 469, True
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=142,level=1,tokens=455), 142, 469, True
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=142,level=1,tokens=455), 142, 469, True
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,719 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=142,level=2,tokens=456), 142, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=144,level=1,tokens=460), 144, 469, True
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=144,level=1,tokens=460), 144, 469, True
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=144,level=1,tokens=460), 144, 469, True
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=144,level=2,tokens=461), 144, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=146,level=1,tokens=465), 146, 469, True
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=146,level=1,tokens=465), 146, 469, True
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=146,level=1,tokens=465), 146, 469, True
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=146,level=0,tokens=466), 146, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=146,level=0,tokens=466), 146, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=146,level=0,tokens=466), 146, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=146,level=0,tokens=466), 146, 469, False
+2023-08-14 13:15:29,720 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=146,level=0,tokens=466), 146, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=146,level=0,tokens=466), 146, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=146,level=0,tokens=466), 146, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=146,level=0,tokens=466), 146, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=148,level=0,tokens=469), 148, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=150,level=0,tokens=472), 150, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=150,level=0,tokens=472), 150, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=150,level=0,tokens=472), 150, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=150,level=0,tokens=472), 150, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=150,level=0,tokens=472), 150, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,721 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=150,level=2,tokens=474), 150, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=152,level=1,tokens=478), 152, 469, True
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=152,level=1,tokens=478), 152, 469, True
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=152,level=1,tokens=478), 152, 469, True
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=152,level=2,tokens=479), 153, 469, True
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=152,level=2,tokens=479), 152, 469, False
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=152,level=2,tokens=479), 153, 469, True
+2023-08-14 13:15:29,722 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=153,level=2,tokens=482), 153, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=153,level=2,tokens=482), 153, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=157,level=1,tokens=484), 157, 469, True
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=157,level=1,tokens=484), 157, 469, True
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=157,level=1,tokens=484), 157, 469, True
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=157,level=2,tokens=485), 158, 469, True
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=157,level=2,tokens=485), 157, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=157,level=2,tokens=485), 158, 469, True
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=158,level=2,tokens=488), 158, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=158,level=2,tokens=488), 158, 469, False
+2023-08-14 13:15:29,723 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=162,level=1,tokens=490), 162, 469, True
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=162,level=1,tokens=490), 162, 469, True
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=162,level=1,tokens=490), 162, 469, True
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=162,level=2,tokens=491), 163, 469, True
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=162,level=2,tokens=491), 162, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=162,level=2,tokens=491), 163, 469, True
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=163,level=2,tokens=494), 163, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=163,level=2,tokens=494), 163, 469, False
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=167,level=1,tokens=496), 167, 469, True
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=167,level=1,tokens=496), 167, 469, True
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=167,level=1,tokens=496), 167, 469, True
+2023-08-14 13:15:29,724 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=167,level=0,tokens=497), 167, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=167,level=0,tokens=497), 167, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=167,level=0,tokens=497), 167, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=167,level=0,tokens=497), 167, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=167,level=0,tokens=497), 167, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=167,level=0,tokens=497), 167, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=167,level=0,tokens=497), 167, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=167,level=0,tokens=497), 167, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=169,level=0,tokens=500), 169, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,725 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=171,level=0,tokens=503), 171, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=173,level=0,tokens=506), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=173,level=0,tokens=506), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=173,level=0,tokens=506), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=173,level=0,tokens=506), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=173,level=0,tokens=506), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=173,level=2,tokens=508), 173, 469, False
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=175,level=1,tokens=512), 175, 469, True
+2023-08-14 13:15:29,726 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=175,level=1,tokens=512), 175, 469, True
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=175,level=1,tokens=512), 175, 469, True
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=175,level=2,tokens=513), 175, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=177,level=1,tokens=517), 177, 469, True
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=177,level=1,tokens=517), 177, 469, True
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=177,level=1,tokens=517), 177, 469, True
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=177,level=0,tokens=518), 177, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=177,level=0,tokens=518), 177, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=177,level=0,tokens=518), 177, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=177,level=0,tokens=518), 177, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=177,level=0,tokens=518), 177, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=177,level=0,tokens=518), 177, 469, False
+2023-08-14 13:15:29,727 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=177,level=0,tokens=518), 177, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=177,level=0,tokens=518), 177, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=179,level=0,tokens=521), 179, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=179,level=0,tokens=521), 179, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=179,level=0,tokens=521), 179, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=179,level=0,tokens=521), 179, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=179,level=0,tokens=521), 179, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=179,level=0,tokens=521), 179, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=179,level=0,tokens=521), 179, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=179,level=0,tokens=521), 179, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=181,level=0,tokens=524), 181, 469, False
+2023-08-14 13:15:29,728 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=183,level=0,tokens=527), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=183,level=0,tokens=527), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=183,level=0,tokens=527), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=183,level=0,tokens=527), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=183,level=0,tokens=527), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=183,level=2,tokens=529), 184, 469, True
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=183,level=2,tokens=529), 183, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=183,level=2,tokens=529), 184, 469, True
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=184,level=2,tokens=532), 184, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=184,level=2,tokens=532), 184, 469, False
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=188,level=1,tokens=534), 188, 469, True
+2023-08-14 13:15:29,729 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=188,level=1,tokens=534), 188, 469, True
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=188,level=1,tokens=534), 188, 469, True
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=188,level=2,tokens=535), 188, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=190,level=1,tokens=539), 190, 469, True
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=190,level=1,tokens=539), 190, 469, True
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=190,level=1,tokens=539), 190, 469, True
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,730 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=190,level=2,tokens=540), 190, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=192,level=1,tokens=544), 192, 469, True
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=192,level=1,tokens=544), 192, 469, True
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=192,level=1,tokens=544), 192, 469, True
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=192,level=2,tokens=545), 193, 469, True
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=192,level=2,tokens=545), 192, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=192,level=2,tokens=545), 193, 469, True
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=193,level=2,tokens=548), 193, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=193,level=2,tokens=548), 193, 469, False
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=197,level=1,tokens=550), 197, 469, True
+2023-08-14 13:15:29,731 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=197,level=1,tokens=550), 197, 469, True
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=197,level=1,tokens=550), 197, 469, True
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=197,level=0,tokens=551), 197, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=197,level=0,tokens=551), 197, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=197,level=0,tokens=551), 197, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=197,level=0,tokens=551), 197, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=197,level=0,tokens=551), 197, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=197,level=0,tokens=551), 197, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=197,level=0,tokens=551), 197, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=197,level=0,tokens=551), 197, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=199,level=0,tokens=554), 199, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=201,level=0,tokens=557), 201, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=201,level=0,tokens=557), 201, 469, False
+2023-08-14 13:15:29,732 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=201,level=0,tokens=557), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=201,level=0,tokens=557), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=201,level=0,tokens=557), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=201,level=2,tokens=559), 201, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=203,level=1,tokens=563), 203, 469, True
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=203,level=1,tokens=563), 203, 469, True
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=203,level=1,tokens=563), 203, 469, True
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,733 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=203,level=2,tokens=564), 204, 469, True
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=203,level=2,tokens=564), 203, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=203,level=2,tokens=564), 204, 469, True
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=204,level=2,tokens=567), 204, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=204,level=2,tokens=567), 204, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=208,level=1,tokens=569), 208, 469, True
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=208,level=1,tokens=569), 208, 469, True
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=208,level=1,tokens=569), 208, 469, True
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,734 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=208,level=2,tokens=570), 209, 469, True
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=208,level=2,tokens=570), 208, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=208,level=2,tokens=570), 209, 469, True
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=209,level=2,tokens=573), 209, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=209,level=2,tokens=573), 209, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=213,level=1,tokens=575), 213, 469, True
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=213,level=1,tokens=575), 213, 469, True
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=213,level=1,tokens=575), 213, 469, True
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=213,level=2,tokens=576), 214, 469, True
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=213,level=2,tokens=576), 213, 469, False
+2023-08-14 13:15:29,735 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=213,level=2,tokens=576), 214, 469, True
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=214,level=2,tokens=579), 214, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=214,level=2,tokens=579), 214, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=218,level=1,tokens=581), 218, 469, True
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=218,level=1,tokens=581), 218, 469, True
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=218,level=1,tokens=581), 218, 469, True
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=218,level=2,tokens=582), 219, 469, True
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=218,level=2,tokens=582), 218, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=218,level=2,tokens=582), 219, 469, True
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=219,level=2,tokens=585), 219, 469, False
+2023-08-14 13:15:29,736 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=219,level=2,tokens=585), 219, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=223,level=2,tokens=586), 223, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=225,level=1,tokens=590), 225, 469, True
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=225,level=1,tokens=590), 225, 469, True
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=225,level=1,tokens=590), 225, 469, True
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=225,level=0,tokens=591), 225, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=225,level=0,tokens=591), 225, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=225,level=0,tokens=591), 225, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=225,level=0,tokens=591), 225, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=225,level=0,tokens=591), 225, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=225,level=0,tokens=591), 225, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=225,level=0,tokens=591), 225, 469, False
+2023-08-14 13:15:29,737 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=225,level=0,tokens=591), 225, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=227,level=0,tokens=594), 227, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=229,level=0,tokens=597), 229, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,738 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=231,level=0,tokens=600), 231, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=233,level=0,tokens=603), 233, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=233,level=0,tokens=603), 233, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=233,level=0,tokens=603), 233, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=233,level=0,tokens=603), 233, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=233,level=0,tokens=603), 233, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=233,level=0,tokens=603), 233, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=233,level=0,tokens=603), 233, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=233,level=0,tokens=603), 233, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=234,level=0,tokens=606), 234, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=234,level=0,tokens=606), 234, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=234,level=0,tokens=606), 234, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=234,level=0,tokens=606), 234, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=234,level=0,tokens=606), 234, 469, False
+2023-08-14 13:15:29,739 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=234,level=0,tokens=606), 234, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=234,level=0,tokens=606), 234, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=234,level=0,tokens=606), 234, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=236,level=0,tokens=609), 236, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=238,level=0,tokens=612), 238, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=238,level=0,tokens=612), 238, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=238,level=0,tokens=612), 238, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=238,level=0,tokens=612), 238, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=238,level=0,tokens=612), 238, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,740 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=238,level=2,tokens=614), 238, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=240,level=1,tokens=618), 240, 469, True
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=240,level=1,tokens=618), 240, 469, True
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=240,level=1,tokens=618), 240, 469, True
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=240,level=2,tokens=619), 240, 469, False
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=242,level=1,tokens=623), 242, 469, True
+2023-08-14 13:15:29,741 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=242,level=1,tokens=623), 242, 469, True
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=242,level=1,tokens=623), 242, 469, True
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=242,level=2,tokens=624), 243, 469, True
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=242,level=2,tokens=624), 242, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=242,level=2,tokens=624), 243, 469, True
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=243,level=2,tokens=627), 243, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=243,level=2,tokens=627), 243, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=248,level=1,tokens=629), 248, 469, True
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=248,level=1,tokens=629), 248, 469, True
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=248,level=1,tokens=629), 248, 469, True
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,742 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=248,level=2,tokens=630), 249, 469, True
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=248,level=2,tokens=630), 248, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=248,level=2,tokens=630), 249, 469, True
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=249,level=2,tokens=633), 249, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=249,level=2,tokens=633), 249, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=254,level=1,tokens=635), 254, 469, True
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=254,level=1,tokens=635), 254, 469, True
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=254,level=1,tokens=635), 254, 469, True
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,743 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=254,level=2,tokens=636), 255, 469, True
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=254,level=2,tokens=636), 254, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=254,level=2,tokens=636), 255, 469, True
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=255,level=2,tokens=639), 255, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=255,level=2,tokens=639), 255, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=259,level=2,tokens=640), 259, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=261,level=2,tokens=643), 261, 469, False
+2023-08-14 13:15:29,744 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=261,level=2,tokens=643), 261, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=265,level=1,tokens=645), 265, 469, True
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=265,level=1,tokens=645), 265, 469, True
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=265,level=1,tokens=645), 265, 469, True
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=265,level=2,tokens=646), 265, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=267,level=1,tokens=650), 267, 469, True
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=267,level=1,tokens=650), 267, 469, True
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=267,level=1,tokens=650), 267, 469, True
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=267,level=0,tokens=651), 267, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=267,level=0,tokens=651), 267, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=267,level=0,tokens=651), 267, 469, False
+2023-08-14 13:15:29,745 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=267,level=0,tokens=651), 267, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=267,level=0,tokens=651), 267, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=267,level=0,tokens=651), 267, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=267,level=0,tokens=651), 267, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=267,level=0,tokens=651), 267, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=269,level=0,tokens=654), 269, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=271,level=0,tokens=657), 271, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=271,level=0,tokens=657), 271, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=271,level=0,tokens=657), 271, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=271,level=0,tokens=657), 271, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=271,level=0,tokens=657), 271, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,746 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=271,level=2,tokens=659), 271, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=273,level=1,tokens=663), 273, 469, True
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=273,level=1,tokens=663), 273, 469, True
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=273,level=1,tokens=663), 273, 469, True
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,747 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=273,level=2,tokens=664), 273, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=275,level=1,tokens=668), 275, 469, True
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=275,level=1,tokens=668), 275, 469, True
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=275,level=1,tokens=668), 275, 469, True
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=275,level=2,tokens=669), 275, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=278,level=1,tokens=673), 278, 469, True
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=278,level=1,tokens=673), 278, 469, True
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=278,level=1,tokens=673), 278, 469, True
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=278,level=0,tokens=674), 278, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=278,level=0,tokens=674), 278, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=278,level=0,tokens=674), 278, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=278,level=0,tokens=674), 278, 469, False
+2023-08-14 13:15:29,748 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=278,level=0,tokens=674), 278, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=278,level=0,tokens=674), 278, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=278,level=0,tokens=674), 278, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=278,level=0,tokens=674), 278, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=279,level=0,tokens=677), 279, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=279,level=0,tokens=677), 279, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=279,level=0,tokens=677), 279, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=279,level=0,tokens=677), 279, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=279,level=0,tokens=677), 279, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=279,level=0,tokens=677), 279, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=279,level=0,tokens=677), 279, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=279,level=0,tokens=677), 279, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=281,level=0,tokens=680), 281, 469, False
+2023-08-14 13:15:29,749 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=283,level=0,tokens=683), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=283,level=0,tokens=683), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=283,level=0,tokens=683), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=283,level=0,tokens=683), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=283,level=0,tokens=683), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=283,level=2,tokens=685), 283, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=285,level=1,tokens=689), 285, 469, True
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=285,level=1,tokens=689), 285, 469, True
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=285,level=1,tokens=689), 285, 469, True
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,750 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=285,level=2,tokens=690), 285, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=287,level=2,tokens=693), 287, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=287,level=2,tokens=693), 287, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=294,level=2,tokens=694), 294, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,751 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=296,level=2,tokens=697), 296, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=298,level=2,tokens=700), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=298,level=2,tokens=700), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=298,level=2,tokens=700), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=298,level=2,tokens=700), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=298,level=2,tokens=700), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=298,level=4,tokens=702), 299, 469, True
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=298,level=4,tokens=702), 299, 469, True
+2023-08-14 13:15:29,752 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=298,level=4,tokens=702), 299, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=298,level=4,tokens=702), 299, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=298,level=4,tokens=702), 298, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=298,level=4,tokens=702), 299, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=298,level=4,tokens=702), 299, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=298,level=4,tokens=702), 299, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=298,level=4,tokens=702), 299, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=299,level=3,tokens=706), 299, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=299,level=3,tokens=706), 299, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=299,level=3,tokens=706), 299, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=299,level=4,tokens=707), 300, 469, True
+2023-08-14 13:15:29,753 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=299,level=4,tokens=707), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=299,level=4,tokens=707), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=299,level=4,tokens=707), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=299,level=4,tokens=707), 299, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=299,level=4,tokens=707), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=299,level=4,tokens=707), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=299,level=4,tokens=707), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=299,level=4,tokens=707), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=300,level=3,tokens=711), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=300,level=3,tokens=711), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=300,level=3,tokens=711), 300, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=300,level=4,tokens=712), 301, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=300,level=4,tokens=712), 301, 469, True
+2023-08-14 13:15:29,754 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=300,level=4,tokens=712), 301, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=300,level=4,tokens=712), 301, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=300,level=4,tokens=712), 300, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=300,level=4,tokens=712), 301, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=300,level=4,tokens=712), 301, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=300,level=4,tokens=712), 301, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=300,level=4,tokens=712), 301, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=301,level=3,tokens=716), 301, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=301,level=3,tokens=716), 301, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=301,level=3,tokens=716), 301, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=301,level=4,tokens=717), 301, 469, False
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=303,level=3,tokens=721), 303, 469, True
+2023-08-14 13:15:29,755 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=303,level=3,tokens=721), 303, 469, True
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=303,level=3,tokens=721), 303, 469, True
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=303,level=2,tokens=722), 303, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=305,level=1,tokens=726), 305, 469, True
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=305,level=1,tokens=726), 305, 469, True
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=305,level=1,tokens=726), 305, 469, True
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=305,level=0,tokens=727), 305, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=305,level=0,tokens=727), 305, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=305,level=0,tokens=727), 305, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=305,level=0,tokens=727), 305, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=305,level=0,tokens=727), 305, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=305,level=0,tokens=727), 305, 469, False
+2023-08-14 13:15:29,756 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=305,level=0,tokens=727), 305, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=305,level=0,tokens=727), 305, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=307,level=0,tokens=730), 307, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=309,level=0,tokens=733), 309, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=309,level=0,tokens=733), 309, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=309,level=0,tokens=733), 309, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=309,level=0,tokens=733), 309, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=309,level=0,tokens=733), 309, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,757 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=309,level=2,tokens=735), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=309,level=2,tokens=735), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=309,level=2,tokens=735), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=309,level=2,tokens=735), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=309,level=2,tokens=735), 309, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=309,level=2,tokens=735), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=309,level=2,tokens=735), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=309,level=2,tokens=735), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=309,level=2,tokens=735), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=310,level=1,tokens=739), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=310,level=1,tokens=739), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=310,level=1,tokens=739), 310, 469, True
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,758 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=310,level=2,tokens=740), 310, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=312,level=1,tokens=744), 312, 469, True
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=312,level=1,tokens=744), 312, 469, True
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=312,level=1,tokens=744), 312, 469, True
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=312,level=0,tokens=745), 312, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=312,level=0,tokens=745), 312, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=312,level=0,tokens=745), 312, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=312,level=0,tokens=745), 312, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=312,level=0,tokens=745), 312, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=312,level=0,tokens=745), 312, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=312,level=0,tokens=745), 312, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=312,level=0,tokens=745), 312, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,759 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=314,level=0,tokens=748), 314, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=316,level=0,tokens=751), 316, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=316,level=0,tokens=751), 316, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=316,level=0,tokens=751), 316, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=316,level=0,tokens=751), 316, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=316,level=0,tokens=751), 316, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=316,level=0,tokens=751), 316, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=316,level=0,tokens=751), 316, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=316,level=0,tokens=751), 316, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,760 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=318,level=0,tokens=754), 318, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=320,level=0,tokens=757), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=320,level=0,tokens=757), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=320,level=0,tokens=757), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=320,level=0,tokens=757), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=320,level=0,tokens=757), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=320,level=2,tokens=759), 321, 469, True
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=320,level=2,tokens=759), 320, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=320,level=2,tokens=759), 321, 469, True
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=321,level=2,tokens=762), 321, 469, False
+2023-08-14 13:15:29,761 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=321,level=2,tokens=762), 321, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=325,level=1,tokens=764), 325, 469, True
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=325,level=1,tokens=764), 325, 469, True
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=325,level=1,tokens=764), 325, 469, True
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=325,level=2,tokens=765), 326, 469, True
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=325,level=2,tokens=765), 325, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=325,level=2,tokens=765), 326, 469, True
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=326,level=2,tokens=768), 326, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=326,level=2,tokens=768), 326, 469, False
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=330,level=1,tokens=770), 330, 469, True
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=330,level=1,tokens=770), 330, 469, True
+2023-08-14 13:15:29,762 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=330,level=1,tokens=770), 330, 469, True
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=330,level=0,tokens=771), 330, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=330,level=0,tokens=771), 330, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=330,level=0,tokens=771), 330, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=330,level=0,tokens=771), 330, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=330,level=0,tokens=771), 330, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=330,level=0,tokens=771), 330, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=330,level=0,tokens=771), 330, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=330,level=0,tokens=771), 330, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=332,level=0,tokens=774), 332, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=334,level=0,tokens=777), 334, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=334,level=0,tokens=777), 334, 469, False
+2023-08-14 13:15:29,763 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=334,level=0,tokens=777), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=334,level=0,tokens=777), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=334,level=0,tokens=777), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=334,level=2,tokens=779), 334, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=336,level=1,tokens=783), 336, 469, True
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=336,level=1,tokens=783), 336, 469, True
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=336,level=1,tokens=783), 336, 469, True
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,764 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=336,level=0,tokens=784), 336, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=338,level=0,tokens=787), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=338,level=0,tokens=787), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=338,level=0,tokens=787), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=338,level=0,tokens=787), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=338,level=0,tokens=787), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=338,level=2,tokens=789), 339, 469, True
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=338,level=2,tokens=789), 338, 469, False
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=338,level=2,tokens=789), 339, 469, True
+2023-08-14 13:15:29,765 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=339,level=2,tokens=792), 339, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=339,level=2,tokens=792), 339, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=343,level=1,tokens=794), 343, 469, True
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=343,level=1,tokens=794), 343, 469, True
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=343,level=1,tokens=794), 343, 469, True
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=343,level=2,tokens=795), 344, 469, True
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=343,level=2,tokens=795), 343, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=343,level=2,tokens=795), 344, 469, True
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=344,level=2,tokens=798), 344, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=344,level=2,tokens=798), 344, 469, False
+2023-08-14 13:15:29,766 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=348,level=1,tokens=800), 348, 469, True
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=348,level=1,tokens=800), 348, 469, True
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=348,level=1,tokens=800), 348, 469, True
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=348,level=0,tokens=801), 348, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=348,level=0,tokens=801), 348, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=348,level=0,tokens=801), 348, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=348,level=0,tokens=801), 348, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=348,level=0,tokens=801), 348, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=348,level=0,tokens=801), 348, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=348,level=0,tokens=801), 348, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=348,level=0,tokens=801), 348, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=350,level=0,tokens=804), 350, 469, False
+2023-08-14 13:15:29,767 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=352,level=0,tokens=807), 352, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=352,level=0,tokens=807), 352, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=361,level=0,tokens=808), 361, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=361,level=0,tokens=808), 361, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=361,level=0,tokens=808), 361, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=361,level=0,tokens=808), 361, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=361,level=0,tokens=808), 361, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=361,level=0,tokens=808), 361, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=361,level=0,tokens=808), 361, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=361,level=0,tokens=808), 361, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,768 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=363,level=0,tokens=811), 363, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=365,level=0,tokens=814), 365, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=365,level=0,tokens=814), 365, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=369,level=0,tokens=815), 369, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=371,level=0,tokens=818), 371, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=371,level=0,tokens=818), 371, 469, False
+2023-08-14 13:15:29,769 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=375,level=0,tokens=819), 375, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=375,level=0,tokens=819), 375, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=375,level=0,tokens=819), 375, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=375,level=0,tokens=819), 375, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=375,level=0,tokens=819), 375, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=375,level=0,tokens=819), 375, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=375,level=0,tokens=819), 375, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=375,level=0,tokens=819), 375, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=377,level=0,tokens=822), 377, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=379,level=0,tokens=825), 379, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=379,level=0,tokens=825), 379, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=383,level=0,tokens=826), 383, 469, False
+2023-08-14 13:15:29,770 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=383,level=0,tokens=826), 383, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=383,level=0,tokens=826), 383, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=383,level=0,tokens=826), 383, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=383,level=0,tokens=826), 383, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=383,level=0,tokens=826), 383, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=383,level=0,tokens=826), 383, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=383,level=0,tokens=826), 383, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=385,level=0,tokens=829), 385, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=387,level=0,tokens=832), 387, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=387,level=0,tokens=832), 387, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=387,level=0,tokens=832), 387, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=387,level=0,tokens=832), 387, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=387,level=0,tokens=832), 387, 469, False
+2023-08-14 13:15:29,771 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=387,level=0,tokens=832), 387, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=387,level=0,tokens=832), 387, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=387,level=0,tokens=832), 387, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=389,level=0,tokens=835), 389, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=391,level=0,tokens=838), 391, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=391,level=0,tokens=838), 391, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=391,level=0,tokens=838), 391, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=391,level=0,tokens=838), 391, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=391,level=0,tokens=838), 391, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=391,level=0,tokens=838), 391, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=391,level=0,tokens=838), 391, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=391,level=0,tokens=838), 391, 469, False
+2023-08-14 13:15:29,772 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=393,level=0,tokens=841), 393, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=395,level=0,tokens=844), 395, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=395,level=0,tokens=844), 395, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=395,level=0,tokens=844), 395, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=395,level=0,tokens=844), 395, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=395,level=0,tokens=844), 395, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=395,level=0,tokens=844), 395, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=395,level=0,tokens=844), 395, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=395,level=0,tokens=844), 395, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,773 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=397,level=0,tokens=847), 397, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=399,level=0,tokens=850), 399, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,774 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=401,level=0,tokens=853), 401, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=403,level=0,tokens=856), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=403,level=0,tokens=856), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=403,level=0,tokens=856), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=403,level=0,tokens=856), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=403,level=0,tokens=856), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=403,level=2,tokens=858), 403, 469, False
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=405,level=1,tokens=862), 405, 469, True
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=405,level=1,tokens=862), 405, 469, True
+2023-08-14 13:15:29,775 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=405,level=1,tokens=862), 405, 469, True
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=405,level=2,tokens=863), 405, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=407,level=1,tokens=867), 407, 469, True
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=407,level=1,tokens=867), 407, 469, True
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=407,level=1,tokens=867), 407, 469, True
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,776 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=407,level=0,tokens=868), 407, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=409,level=0,tokens=871), 409, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=411,level=0,tokens=874), 411, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=411,level=0,tokens=874), 411, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=411,level=0,tokens=874), 411, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=411,level=0,tokens=874), 411, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=411,level=0,tokens=874), 411, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=411,level=0,tokens=874), 411, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=411,level=0,tokens=874), 411, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=411,level=0,tokens=874), 411, 469, False
+2023-08-14 13:15:29,777 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=413,level=0,tokens=877), 413, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=415,level=0,tokens=880), 415, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=415,level=0,tokens=880), 415, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,778 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=423,level=0,tokens=881), 423, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=425,level=0,tokens=884), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=425,level=0,tokens=884), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=425,level=0,tokens=884), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=425,level=0,tokens=884), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=425,level=0,tokens=884), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=425,level=2,tokens=886), 426, 469, True
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=425,level=2,tokens=886), 426, 469, True
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=425,level=2,tokens=886), 426, 469, True
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=425,level=2,tokens=886), 426, 469, True
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=425,level=2,tokens=886), 425, 469, False
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=425,level=2,tokens=886), 426, 469, True
+2023-08-14 13:15:29,779 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=425,level=2,tokens=886), 426, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=425,level=2,tokens=886), 426, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=425,level=2,tokens=886), 426, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=426,level=1,tokens=890), 426, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=426,level=1,tokens=890), 426, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=426,level=1,tokens=890), 426, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=426,level=2,tokens=891), 427, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=426,level=2,tokens=891), 427, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=426,level=2,tokens=891), 427, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=426,level=2,tokens=891), 427, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=426,level=2,tokens=891), 426, 469, False
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=426,level=2,tokens=891), 427, 469, True
+2023-08-14 13:15:29,780 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=426,level=2,tokens=891), 427, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=426,level=2,tokens=891), 427, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=426,level=2,tokens=891), 427, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=427,level=1,tokens=895), 427, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=427,level=1,tokens=895), 427, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=427,level=1,tokens=895), 427, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=427,level=2,tokens=896), 428, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=427,level=2,tokens=896), 428, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=427,level=2,tokens=896), 428, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=427,level=2,tokens=896), 428, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=427,level=2,tokens=896), 427, 469, False
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=427,level=2,tokens=896), 428, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=427,level=2,tokens=896), 428, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=427,level=2,tokens=896), 428, 469, True
+2023-08-14 13:15:29,781 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=427,level=2,tokens=896), 428, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=428,level=1,tokens=900), 428, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=428,level=1,tokens=900), 428, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=428,level=1,tokens=900), 428, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=428,level=2,tokens=901), 429, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=428,level=2,tokens=901), 429, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=428,level=2,tokens=901), 429, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=428,level=2,tokens=901), 429, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=428,level=2,tokens=901), 428, 469, False
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=428,level=2,tokens=901), 429, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=428,level=2,tokens=901), 429, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=428,level=2,tokens=901), 429, 469, True
+2023-08-14 13:15:29,782 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=428,level=2,tokens=901), 429, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=429,level=1,tokens=905), 429, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=429,level=1,tokens=905), 429, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=429,level=1,tokens=905), 429, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=429,level=2,tokens=906), 430, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=429,level=2,tokens=906), 430, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=429,level=2,tokens=906), 430, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=429,level=2,tokens=906), 430, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=429,level=2,tokens=906), 429, 469, False
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=429,level=2,tokens=906), 430, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=429,level=2,tokens=906), 430, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=429,level=2,tokens=906), 430, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=429,level=2,tokens=906), 430, 469, True
+2023-08-14 13:15:29,783 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=430,level=1,tokens=910), 430, 469, True
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=430,level=1,tokens=910), 430, 469, True
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=430,level=1,tokens=910), 430, 469, True
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=430,level=2,tokens=911), 430, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=432,level=1,tokens=915), 432, 469, True
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=432,level=1,tokens=915), 432, 469, True
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=432,level=1,tokens=915), 432, 469, True
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,784 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,786 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,786 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,786 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,786 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,786 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=432,level=0,tokens=916), 432, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=434,level=0,tokens=919), 434, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=434,level=0,tokens=919), 434, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=434,level=0,tokens=919), 434, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=434,level=0,tokens=919), 434, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=434,level=0,tokens=919), 434, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=434,level=0,tokens=919), 434, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=434,level=0,tokens=919), 434, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=434,level=0,tokens=919), 434, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=436,level=0,tokens=922), 436, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=438,level=0,tokens=925), 438, 469, False
+2023-08-14 13:15:29,787 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=438,level=0,tokens=925), 438, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=438,level=0,tokens=925), 438, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=438,level=0,tokens=925), 438, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=438,level=0,tokens=925), 438, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=438,level=0,tokens=925), 438, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=438,level=0,tokens=925), 438, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=438,level=0,tokens=925), 438, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=440,level=0,tokens=928), 440, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=442,level=0,tokens=931), 442, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=442,level=0,tokens=931), 442, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=442,level=0,tokens=931), 442, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=442,level=0,tokens=931), 442, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=442,level=0,tokens=931), 442, 469, False
+2023-08-14 13:15:29,788 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=442,level=0,tokens=931), 442, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=442,level=0,tokens=931), 442, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=442,level=0,tokens=931), 442, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=444,level=0,tokens=934), 444, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=446,level=0,tokens=937), 446, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=446,level=0,tokens=937), 446, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=446,level=0,tokens=937), 446, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=446,level=0,tokens=937), 446, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=446,level=0,tokens=937), 446, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=446,level=0,tokens=937), 446, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=446,level=0,tokens=937), 446, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=446,level=0,tokens=937), 446, 469, False
+2023-08-14 13:15:29,789 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=448,level=0,tokens=940), 448, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=450,level=0,tokens=943), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=450,level=0,tokens=943), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=450,level=0,tokens=943), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=450,level=0,tokens=943), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=450,level=0,tokens=943), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,790 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=450,level=2,tokens=945), 450, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=452,level=1,tokens=949), 452, 469, True
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=452,level=1,tokens=949), 452, 469, True
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=452,level=1,tokens=949), 452, 469, True
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=452,level=2,tokens=950), 453, 469, True
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=452,level=2,tokens=950), 452, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=452,level=2,tokens=950), 453, 469, True
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=453,level=2,tokens=953), 453, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=453,level=2,tokens=953), 453, 469, False
+2023-08-14 13:15:29,791 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=458,level=2,tokens=954), 458, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=460,level=1,tokens=958), 460, 469, True
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=460,level=1,tokens=958), 460, 469, True
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=460,level=1,tokens=958), 460, 469, True
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=460,level=0,tokens=959), 460, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=460,level=0,tokens=959), 460, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=460,level=0,tokens=959), 460, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=460,level=0,tokens=959), 460, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=460,level=0,tokens=959), 460, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=460,level=0,tokens=959), 460, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=460,level=0,tokens=959), 460, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=460,level=0,tokens=959), 460, 469, False
+2023-08-14 13:15:29,792 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=462,level=0,tokens=962), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=462,level=0,tokens=962), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=462,level=0,tokens=962), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=462,level=0,tokens=962), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=462,level=0,tokens=962), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=462,level=2,tokens=964), 463, 469, True
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=462,level=2,tokens=964), 463, 469, True
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=462,level=2,tokens=964), 463, 469, True
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=462,level=2,tokens=964), 463, 469, True
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=462,level=2,tokens=964), 462, 469, False
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=462,level=2,tokens=964), 463, 469, True
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=462,level=2,tokens=964), 463, 469, True
+2023-08-14 13:15:29,793 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=462,level=2,tokens=964), 463, 469, True
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=462,level=2,tokens=964), 463, 469, True
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=463,level=2,tokens=967), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=463,level=2,tokens=967), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=463,level=2,tokens=967), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=463,level=2,tokens=967), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=463,level=2,tokens=967), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=463,level=4,tokens=969), 464, 469, True
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=463,level=4,tokens=969), 464, 469, True
+2023-08-14 13:15:29,794 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=463,level=4,tokens=969), 464, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=463,level=4,tokens=969), 464, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=463,level=4,tokens=969), 463, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=463,level=4,tokens=969), 464, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=463,level=4,tokens=969), 464, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=463,level=4,tokens=969), 464, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=463,level=4,tokens=969), 464, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=464,level=3,tokens=973), 464, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=464,level=3,tokens=973), 464, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=464,level=3,tokens=973), 464, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=464,level=4,tokens=974), 465, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=464,level=4,tokens=974), 465, 469, True
+2023-08-14 13:15:29,795 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=464,level=4,tokens=974), 465, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=464,level=4,tokens=974), 465, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=464,level=4,tokens=974), 464, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=464,level=4,tokens=974), 465, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=464,level=4,tokens=974), 465, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=464,level=4,tokens=974), 465, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=464,level=4,tokens=974), 465, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=465,level=3,tokens=978), 465, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=465,level=3,tokens=978), 465, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=465,level=3,tokens=978), 465, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=465,level=4,tokens=979), 466, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=465,level=4,tokens=979), 466, 469, True
+2023-08-14 13:15:29,796 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=465,level=4,tokens=979), 466, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=465,level=4,tokens=979), 466, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=465,level=4,tokens=979), 465, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=465,level=4,tokens=979), 466, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=465,level=4,tokens=979), 466, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=465,level=4,tokens=979), 466, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=465,level=4,tokens=979), 466, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=466,level=3,tokens=983), 466, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=466,level=3,tokens=983), 466, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=466,level=3,tokens=983), 466, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=466,level=4,tokens=984), 467, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=466,level=4,tokens=984), 467, 469, True
+2023-08-14 13:15:29,797 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=466,level=4,tokens=984), 467, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=466,level=4,tokens=984), 467, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=466,level=4,tokens=984), 466, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=466,level=4,tokens=984), 467, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=466,level=4,tokens=984), 467, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=466,level=4,tokens=984), 467, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=466,level=4,tokens=984), 467, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=467,level=3,tokens=988), 467, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=467,level=3,tokens=988), 467, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=467,level=3,tokens=988), 467, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=467,level=4,tokens=989), 468, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=467,level=4,tokens=989), 468, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=467,level=4,tokens=989), 468, 469, True
+2023-08-14 13:15:29,798 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=467,level=4,tokens=989), 468, 469, True
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=467,level=4,tokens=989), 467, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=467,level=4,tokens=989), 468, 469, True
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=467,level=4,tokens=989), 468, 469, True
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=467,level=4,tokens=989), 468, 469, True
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=467,level=4,tokens=989), 468, 469, True
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=468,level=3,tokens=993), 468, 469, True
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=468,level=3,tokens=993), 468, 469, True
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=468,level=3,tokens=993), 468, 469, True
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/code.py | entering code: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/fence.py | entering fence: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/blockquote.py | entering blockquote: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/hr.py | entering hr: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/list.py | entering list: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/reference.py | entering reference: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/html_block.py | entering html_block: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/heading.py | entering heading: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/lheading.py | entering lheading: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,799 | DEBUG | /usr/local/lib/python3.10/dist-packages/markdown_it/rules_block/paragraph.py | entering paragraph: StateBlock(line=468,level=4,tokens=994), 468, 469, False
+2023-08-14 13:15:29,896 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | https://api.gradio.app:443 "POST /gradio-initiated-analytics/ HTTP/1.1" 200 None
+2023-08-14 13:15:30,017 | DEBUG | /usr/lib/python3.10/asyncio/selector_events.py | Using selector: EpollSelector
+2023-08-14 13:15:30,033 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | Starting new HTTP connection (1): 127.0.0.1:7860
+2023-08-14 13:15:30,036 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | http://127.0.0.1:7860 "GET /startup-events HTTP/1.1" 200 5
+2023-08-14 13:15:30,038 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | Starting new HTTP connection (1): 127.0.0.1:7860
+2023-08-14 13:15:30,070 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | http://127.0.0.1:7860 "HEAD / HTTP/1.1" 200 0
+2023-08-14 13:15:30,073 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | Starting new HTTPS connection (1): api.gradio.app:443
+2023-08-14 13:15:34,534 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | https://api.gradio.app:443 "GET /v2/tunnel-request HTTP/1.1" 200 None
+2023-08-14 13:15:34,536 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | Starting new HTTPS connection (1): cdn-media.huggingface.co:443
+2023-08-14 13:15:34,647 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | https://cdn-media.huggingface.co:443 "GET /frpc-gradio-0.2/frpc_linux_amd64 HTTP/1.1" 200 11374592
+2023-08-14 13:15:35,268 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | Starting new HTTPS connection (1): api.gradio.app:443
+2023-08-14 13:15:36,438 | DEBUG | /usr/local/lib/python3.10/dist-packages/urllib3/connectionpool.py | https://api.gradio.app:443 "POST /gradio-launched-telemetry/ HTTP/1.1" 200 None
+2023-08-14 13:30:12,307 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:30:12,340 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:30:30,374 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:30:30,375 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:30:36,817 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:30:36,818 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:30:45,851 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:30:45,852 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:30:50,418 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:30:50,419 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:30:55,034 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:30:55,035 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:31:14,911 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:31:14,912 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:31:20,635 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:31:20,636 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:31:20,873 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:31:20,874 | INFO | /content/kohya_ss/dreambooth_gui.py | Start training Dreambooth...
+2023-08-14 13:31:20,877 | INFO | /content/kohya_ss/library/common_gui.py | Image folder does not exist
+2023-08-14 13:31:20,879 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:31:31,669 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:31:31,670 | INFO | /content/kohya_ss/dreambooth_gui.py | Start training Dreambooth...
+2023-08-14 13:31:31,672 | INFO | /content/kohya_ss/library/common_gui.py | Image folder does not exist
+2023-08-14 13:31:31,673 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:31:33,574 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:31:33,575 | INFO | /content/kohya_ss/dreambooth_gui.py | Start training Dreambooth...
+2023-08-14 13:31:33,577 | INFO | /content/kohya_ss/library/common_gui.py | Image folder does not exist
+2023-08-14 13:31:33,580 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:33:51,467 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:33:51,469 | INFO | /content/kohya_ss/dreambooth_gui.py | Start training Dreambooth...
+2023-08-14 13:33:51,473 | ERROR | /content/kohya_ss/library/common_gui.py | The following folders do not match the required pattern <number>_<text>: /content/drive/.file-revisions-by-id, /content/drive/.Trash-0, /content/drive/.shortcut-targets-by-id, /content/drive/MyDrive
+2023-08-14 13:33:51,478 | ERROR | /content/kohya_ss/library/common_gui.py | Please follow the folder structure documentation found at docs\image_folder_structure.md ...
+2023-08-14 13:33:51,480 | INFO | /content/kohya_ss/library/common_gui.py | Regularisation folder does not exist
+2023-08-14 13:33:51,482 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:35:45,881 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:35:45,885 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend module://ipykernel.pylab.backend_inline version unknown.
+2023-08-14 13:35:45,883 | INFO | /content/kohya_ss/dreambooth_gui.py | Start training Dreambooth...
+2023-08-14 13:35:45,890 | INFO | /content/kohya_ss/library/common_gui.py | Valid image folder names found in: /content/drive/MyDrive/images
+2023-08-14 13:35:45,891 | INFO | /content/kohya_ss/library/common_gui.py | Regularisation folder does not exist
+2023-08-14 13:35:45,893 | DEBUG | /usr/local/lib/python3.10/dist-packages/matplotlib/pyplot.py | Loaded backend agg version v2.2.
+2023-08-14 13:36:09,652 | INFO | /content/kohya_ss/dreambooth_gui.py | Start training Dreambooth...
+2023-08-14 13:36:09,656 | ERROR | /content/kohya_ss/library/common_gui.py | The following folders do not match the required pattern <number>_<text>: /content/drive/MyDrive/dataset, /content/drive/MyDrive/log, /content/drive/MyDrive/.ipynb_checkpoints, /content/drive/MyDrive/stable_diffusion_weights, /content/drive/MyDrive/diffusion_model, /content/drive/MyDrive/Colab Notebooks, /content/drive/MyDrive/images
+2023-08-14 13:36:09,659 | ERROR | /content/kohya_ss/library/common_gui.py | Please follow the folder structure documentation found at docs\image_folder_structure.md ...
+2023-08-14 13:36:09,660 | INFO | /content/kohya_ss/library/common_gui.py | Regularisation folder does not exist
+2023-08-14 13:36:29,010 | INFO | /content/kohya_ss/dreambooth_gui.py | Start training Dreambooth...
+2023-08-14 13:36:29,013 | INFO | /content/kohya_ss/library/common_gui.py | Valid image folder names found in: /content/drive/MyDrive/images
+2023-08-14 13:36:29,015 | INFO | /content/kohya_ss/library/common_gui.py | Regularisation folder does not exist
+2023-08-14 13:39:49,115 | INFO | /content/kohya_ss/dreambooth_gui.py | Start training Dreambooth...
+2023-08-14 13:39:49,117 | INFO | /content/kohya_ss/library/common_gui.py | Valid image folder names found in: /content/drive/MyDrive/images
+2023-08-14 13:39:49,473 | ERROR | /content/kohya_ss/library/common_gui.py | No image folders found in /content/drive/MyDrive/lora-models. Please follow the folder structure documentation found at docs\image_folder_structure.md ...
+2023-08-14 13:39:49,475 | INFO | /content/kohya_ss/library/common_gui.py | Output folder path is missing
+2023-08-14 13:40:40,445 | INFO | /content/kohya_ss/dreambooth_gui.py | Start training Dreambooth...
+2023-08-14 13:40:40,451 | INFO | /content/kohya_ss/library/common_gui.py | Valid image folder names found in: /content/drive/MyDrive/images
+2023-08-14 13:40:40,453 | INFO | /content/kohya_ss/library/common_gui.py | Headless mode, skipping verification if model already exist... if model already exist it will be overwritten...
+2023-08-14 13:40:40,455 | INFO | /content/kohya_ss/dreambooth_gui.py | Folder 100_test : steps 800
+2023-08-14 13:40:40,457 | INFO | /content/kohya_ss/dreambooth_gui.py | max_train_steps = 800
+2023-08-14 13:40:40,458 | INFO | /content/kohya_ss/dreambooth_gui.py | stop_text_encoder_training = 0
+2023-08-14 13:40:40,460 | INFO | /content/kohya_ss/dreambooth_gui.py | lr_warmup_steps = 80
+2023-08-14 13:40:40,464 | INFO | /content/kohya_ss/dreambooth_gui.py | Saving training config to /content/drive/MyDrive/lora-models/last_20230814-134040.json...
+2023-08-14 13:40:40,471 | INFO | /content/kohya_ss/dreambooth_gui.py | accelerate launch --num_cpu_threads_per_process=2 "train_db.py" --enable_bucket --pretrained_model_name_or_path="runwayml/stable-diffusion-v1-5" --train_data_dir="/content/drive/MyDrive/images" --resolution="512,512" --output_dir="/content/drive/MyDrive/lora-models" --logging_dir="/content/drive/MyDrive/log" --save_model_as=safetensors --output_name="last" --max_data_loader_n_workers="0" --learning_rate="1e-05" --lr_scheduler="cosine" --lr_warmup_steps="80" --train_batch_size="1" --max_train_steps="800" --save_every_n_epochs="1" --mixed_precision="fp16" --save_precision="fp16" --cache_latents --optimizer_type="AdamW8bit" --max_data_loader_n_workers="0" --bucket_reso_steps=64 --xformers --bucket_no_upscale
diff --git a/setup.ps1 b/setup.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..df5f92e6802ae8e82bc6961e520d3ab9df0efc42
--- /dev/null
+++ b/setup.ps1
@@ -0,0 +1,27 @@
+
+# Check if Python version meets the recommended version
+$pythonVersion = & .\venv\Scripts\python.exe --version 2>$null
+if ($pythonVersion -notmatch "^Python $PYTHON_VER") {
+    Write-Host "Warning: Python version $PYTHON_VER is recommended."
+}
+
+if (-not (Test-Path -Path "venv")) {
+    Write-Host "Creating venv..."
+    python -m venv venv
+}
+
+# Create the directory if it doesn't exist
+$null = New-Item -ItemType Directory -Force -Path ".\logs\setup"
+
+# Deactivate the virtual environment
+& .\venv\Scripts\deactivate.bat
+
+# Calling external python program to check for local modules
+& .\venv\Scripts\python.exe .\setup\check_local_modules.py
+
+& .\venv\Scripts\activate.bat
+
+& .\venv\Scripts\python.exe .\setup\setup_windows.py
+
+# Deactivate the virtual environment
+& .\venv\Scripts\deactivate.bat
diff --git a/setup.sh b/setup.sh
new file mode 100644
index 0000000000000000000000000000000000000000..3f521f37cd446b62a022ecf29542799a70c3a528
--- /dev/null
+++ b/setup.sh
@@ -0,0 +1,624 @@
+#!/usr/bin/env bash
+
+# Function to display help information
+display_help() {
+  cat <<EOF
+Kohya_SS Installation Script for POSIX operating systems.
+
+Usage:
+  # Specifies custom branch, install directory, and git repo
+  setup.sh -b dev -d /workspace/kohya_ss -g https://mycustom.repo.tld/custom_fork.git
+
+  # Same as example 1, but uses long options
+  setup.sh --branch=dev --dir=/workspace/kohya_ss --git-repo=https://mycustom.repo.tld/custom_fork.git
+
+  # Maximum verbosity, fully automated installation in a runpod environment skipping the runpod env checks
+  setup.sh -vvv --skip-space-check --runpod
+
+Options:
+  -b BRANCH, --branch=BRANCH    Select which branch of kohya to check out on new installs.
+  -d DIR, --dir=DIR             The full path you want kohya_ss installed to.
+  -g REPO, --git_repo=REPO      You can optionally provide a git repo to check out for runpod installation. Useful for custom forks.
+  -h, --help                    Show this screen.
+  -i, --interactive             Interactively configure accelerate instead of using default config file.
+  -n, --no-git-update           Do not update kohya_ss repo. No git pull or clone operations.
+  -p, --public                  Expose public URL in runpod mode. Won't have an effect in other modes.
+  -r, --runpod                  Forces a runpod installation. Useful if detection fails for any reason.
+  -s, --skip-space-check        Skip the 10Gb minimum storage space check.
+  -u, --no-gui                  Skips launching the GUI.
+  -v, --verbose                 Increase verbosity levels up to 3.
+EOF
+}
+
+# Helper function to check if variable is set and non-empty
+env_var_exists() {
+  if [[ -n "${!1}" ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# Check if RUNPOD variable should be set
+RUNPOD=false
+if env_var_exists RUNPOD_POD_ID || env_var_exists RUNPOD_API_KEY; then
+  RUNPOD=true
+fi
+
+# Directory of the script
+SCRIPT_DIR="$(cd -- $(dirname -- "$0") && pwd)"
+
+# Variables defined before the getopts loop, so we have sane default values.
+# Default installation locations based on OS and environment
+if [[ "$OSTYPE" == "lin"* ]]; then
+  if [ "$RUNPOD" = true ]; then
+    DIR="/workspace/kohya_ss"
+  elif [ -d "$SCRIPT_DIR/.git" ]; then
+    DIR="$SCRIPT_DIR"
+  elif [ -w "/opt" ]; then
+    DIR="/opt/kohya_ss"
+  elif env_var_exists HOME; then
+    DIR="$HOME/kohya_ss"
+  else
+    # The last fallback is simply PWD
+    DIR="$(PWD)"
+  fi
+else
+  if [ -d "$SCRIPT_DIR/.git" ]; then
+    DIR="$SCRIPT_DIR"
+  elif env_var_exists HOME; then
+    DIR="$HOME/kohya_ss"
+  else
+    # The last fallback is simply PWD
+    DIR="$(PWD)"
+  fi
+fi
+
+# Variables
+BRANCH="master"
+GIT_REPO="https://github.com/bmaltais/kohya_ss.git"
+INTERACTIVE=false
+PUBLIC=false
+SKIP_SPACE_CHECK=false
+SKIP_GIT_UPDATE=true
+SKIP_GUI=false
+VERBOSITY=2
+MAXVERBOSITY=6
+DIR=""
+PARENT_DIR=""
+VENV_DIR=""
+
+# Function to get the distro name
+get_distro_name() {
+  local line
+  if [ -f /etc/os-release ]; then
+    # We search for the line starting with ID=
+    # Then we remove the ID= prefix to get the name itself
+    line="$(grep -Ei '^ID=' /etc/os-release)"
+    echo "Raw detected os-release distro line: $line" >&5
+    line=${line##*=}
+    echo "$line"
+    return 0
+  elif command -v python >/dev/null; then
+    line="$(python -mplatform)"
+    echo "$line"
+    return 0
+  elif command -v python3 >/dev/null; then
+    line="$(python3 -mplatform)"
+    echo "$line"
+    return 0
+  else
+    line="None"
+    echo "$line"
+    return 1
+  fi
+}
+
+# Function to get the distro family
+get_distro_family() {
+  local line
+  if [ -f /etc/os-release ]; then
+    if grep -Eiq '^ID_LIKE=' /etc/os-release >/dev/null; then
+      line="$(grep -Ei '^ID_LIKE=' /etc/os-release)"
+      echo "Raw detected os-release distro family line: $line" >&5
+      line=${line##*=}
+      echo "$line"
+      return 0
+    else
+      line="None"
+      echo "$line"
+      return 1
+    fi
+  else
+    line="None"
+    echo "$line"
+    return 1
+  fi
+}
+
+# Function to check available storage space
+check_storage_space() {
+  if [ "$SKIP_SPACE_CHECK" = false ]; then
+    if [ "$(size_available)" -lt 10 ]; then
+      echo "You have less than 10Gb of free space. This installation may fail."
+      MSGTIMEOUT=10 # In seconds
+      MESSAGE="Continuing in..."
+      echo "Press control-c to cancel the installation."
+      for ((i = MSGTIMEOUT; i >= 0; i--)); do
+        printf "\r${MESSAGE} %ss. " "${i}"
+        sleep 1
+      done
+    fi
+  fi
+}
+
+# Function to create symlinks
+create_symlinks() {
+  local symlink="$1"
+  local target_file="$2"
+
+  echo "Checking symlinks now."
+
+  # Check if the symlink exists
+  if [ -L "$symlink" ]; then
+    # Check if the linked file exists and points to the expected file
+    if [ -e "$symlink" ] && [ "$(readlink "$symlink")" == "$target_file" ]; then
+      echo "$(basename "$symlink") symlink looks fine. Skipping."
+    else
+      if [ -f "$target_file" ]; then
+        echo "Broken symlink detected. Recreating $(basename "$symlink")."
+        rm "$symlink" && ln -s "$target_file" "$symlink"
+      else
+        echo "$target_file does not exist. Nothing to link."
+      fi
+    fi
+  else
+    echo "Linking $(basename "$symlink")."
+    ln -s "$target_file" "$symlink"
+  fi
+}
+
+# Function to install Python dependencies
+install_python_dependencies() {
+  local TEMP_REQUIREMENTS_FILE
+
+  # Switch to local virtual env
+  echo "Switching to virtual Python environment."
+  if ! inDocker; then
+    if command -v python3.10 >/dev/null; then
+      python3.10 -m venv "$DIR/venv"
+    elif command -v python3 >/dev/null; then
+      python3 -m venv "$DIR/venv"
+    else
+      echo "Valid python3 or python3.10 binary not found."
+      echo "Cannot proceed with the python steps."
+      return 1
+    fi
+
+    # Activate the virtual environment
+    source "$DIR/venv/bin/activate"
+  fi
+
+  case "$OSTYPE" in
+    "lin"*)
+      if [ "$RUNPOD" = true ]; then
+        python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_runpod.txt
+      else
+        python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_linux.txt
+      fi
+      ;;
+    "darwin"*)
+      if [[ "$(uname -m)" == "arm64" ]]; then
+        python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_macos_arm64.txt
+      else
+        python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_macos_amd64.txt
+      fi
+      ;;
+  esac
+
+  if [ -n "$VIRTUAL_ENV" ] && ! inDocker; then
+    if command -v deactivate >/dev/null; then
+      echo "Exiting Python virtual environment."
+      deactivate
+    else
+      echo "deactivate command not found. Could still be in the Python virtual environment."
+    fi
+  fi
+}
+
+# Function to configure accelerate
+configure_accelerate() {
+  echo "Source accelerate config location: $DIR/config_files/accelerate/default_config.yaml" >&3
+  if [ "$INTERACTIVE" = true ]; then
+    accelerate config
+  else
+    if env_var_exists HF_HOME; then
+      if [ ! -f "$HF_HOME/accelerate/default_config.yaml" ]; then
+        mkdir -p "$HF_HOME/accelerate/" &&
+          echo "Target accelerate config location: $HF_HOME/accelerate/default_config.yaml" >&3
+        cp "$DIR/config_files/accelerate/default_config.yaml" "$HF_HOME/accelerate/default_config.yaml" &&
+          echo "Copied accelerate config file to: $HF_HOME/accelerate/default_config.yaml"
+      fi
+    elif env_var_exists XDG_CACHE_HOME; then
+      if [ ! -f "$XDG_CACHE_HOME/huggingface/accelerate" ]; then
+        mkdir -p "$XDG_CACHE_HOME/huggingface/accelerate" &&
+          echo "Target accelerate config location: $XDG_CACHE_HOME/accelerate/default_config.yaml" >&3
+        cp "$DIR/config_files/accelerate/default_config.yaml" "$XDG_CACHE_HOME/huggingface/accelerate/default_config.yaml" &&
+          echo "Copied accelerate config file to: $XDG_CACHE_HOME/huggingface/accelerate/default_config.yaml"
+      fi
+    elif env_var_exists HOME; then
+      if [ ! -f "$HOME/.cache/huggingface/accelerate" ]; then
+        mkdir -p "$HOME/.cache/huggingface/accelerate" &&
+          echo "Target accelerate config location: $HOME/accelerate/default_config.yaml" >&3
+        cp "$DIR/config_files/accelerate/default_config.yaml" "$HOME/.cache/huggingface/accelerate/default_config.yaml" &&
+          echo "Copying accelerate config file to: $HOME/.cache/huggingface/accelerate/default_config.yaml"
+      fi
+    else
+      echo "Could not place the accelerate configuration file. Please configure manually."
+      sleep 2
+      accelerate config
+    fi
+  fi
+}
+
+# Function to update Kohya_SS repo
+update_kohya_ss() {
+  if [ "$SKIP_GIT_UPDATE" = false ]; then
+    if command -v git >/dev/null; then
+      # First, we make sure there are no changes that need to be made in git, so no work is lost.
+      if [ "$(git -C "$DIR" status --porcelain=v1 2>/dev/null | wc -l)" -gt 0 ] &&
+        echo "These files need to be committed or discarded: " >&4 &&
+        git -C "$DIR" status >&4; then
+        echo "There are changes that need to be committed or discarded in the repo in $DIR."
+        echo "Commit those changes or run this script with -n to skip git operations entirely."
+        exit 1
+      fi
+
+      echo "Attempting to clone $GIT_REPO."
+      if [ ! -d "$DIR/.git" ]; then
+        echo "Cloning and switching to $GIT_REPO:$BRANCH" >&4
+        git -C "$PARENT_DIR" clone -b "$BRANCH" "$GIT_REPO" "$(basename "$DIR")" >&3
+        git -C "$DIR" switch "$BRANCH" >&4
+      else
+        echo "git repo detected. Attempting to update repository instead."
+        echo "Updating: $GIT_REPO"
+        git -C "$DIR" pull "$GIT_REPO" "$BRANCH" >&3
+        if ! git -C "$DIR" switch "$BRANCH" >&4; then
+          echo "Branch $BRANCH did not exist. Creating it." >&4
+          git -C "$DIR" switch -c "$BRANCH" >&4
+        fi
+      fi
+    else
+      echo "You need to install git."
+      echo "Rerun this after installing git or run this script with -n to skip the git operations."
+    fi
+  else
+    echo "Skipping git operations."
+  fi
+}
+
+# Section: Command-line options parsing
+
+while getopts ":vb:d:g:inprus-:" opt; do
+  # support long options: https://stackoverflow.com/a/28466267/519360
+  if [ "$opt" = "-" ]; then # long option: reformulate OPT and OPTARG
+    opt="${OPTARG%%=*}"     # extract long option name
+    OPTARG="${OPTARG#$opt}" # extract long option argument (may be empty)
+    OPTARG="${OPTARG#=}"    # if long option argument, remove assigning `=`
+  fi
+  
+  case $opt in
+  b | branch) BRANCH="$OPTARG" ;;
+  d | dir) DIR="$OPTARG" ;;
+  g | git-repo) GIT_REPO="$OPTARG" ;;
+  i | interactive) INTERACTIVE=true ;;
+  n | no-git-update) SKIP_GIT_UPDATE=true ;;
+  p | public) PUBLIC=true ;;
+  r | runpod) RUNPOD=true ;;
+  s | skip-space-check) SKIP_SPACE_CHECK=true ;;
+  u | no-gui) SKIP_GUI=true ;;
+  v) ((VERBOSITY = VERBOSITY + 1)) ;;
+  h) display_help && exit 0 ;;
+  *) display_help && exit 0 ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+# Just in case someone puts in a relative path into $DIR,
+# we're going to get the absolute path of that.
+if [[ "$DIR" != /* ]] && [[ "$DIR" != ~* ]]; then
+  DIR="$(
+    cd "$(dirname "$DIR")" || exit 1
+    pwd
+  )/$(basename "$DIR")"
+fi
+
+for v in $( #Start counting from 3 since 1 and 2 are standards (stdout/stderr).
+  seq 3 $VERBOSITY
+); do
+  (("$v" <= "$MAXVERBOSITY")) && eval exec "$v>&2" #Don't change anything higher than the maximum verbosity allowed.
+done
+
+for v in $( #From the verbosity level one higher than requested, through the maximum;
+  seq $((VERBOSITY + 1)) $MAXVERBOSITY
+); do
+  (("$v" > "2")) && eval exec "$v>/dev/null" #Redirect these to bitbucket, provided that they don't match stdout and stderr.
+done
+
+# Example of how to use the verbosity levels.
+# printf "%s\n" "This message is seen at verbosity level 1 and above." >&3
+# printf "%s\n" "This message is seen at verbosity level 2 and above." >&4
+# printf "%s\n" "This message is seen at verbosity level 3 and above." >&5
+
+# Debug variable dump at max verbosity
+echo "BRANCH: $BRANCH
+DIR: $DIR
+GIT_REPO: $GIT_REPO
+INTERACTIVE: $INTERACTIVE
+PUBLIC: $PUBLIC
+RUNPOD: $RUNPOD
+SKIP_SPACE_CHECK: $SKIP_SPACE_CHECK
+VERBOSITY: $VERBOSITY
+Script directory is ${SCRIPT_DIR}." >&5
+
+# This must be set after the getopts loop to account for $DIR changes.
+PARENT_DIR="$(dirname "${DIR}")"
+VENV_DIR="$DIR/venv"
+
+if [ -w "$PARENT_DIR" ] && [ ! -d "$DIR" ]; then
+  echo "Creating install folder ${DIR}."
+  mkdir "$DIR"
+fi
+
+if [ ! -w "$DIR" ]; then
+  echo "We cannot write to ${DIR}."
+  echo "Please ensure the install directory is accurate and you have the correct permissions."
+  exit 1
+fi
+
+# Shared functions
+# This checks for free space on the installation drive and returns that in Gb.
+size_available() {
+  local folder
+  if [ -d "$DIR" ]; then
+    folder="$DIR"
+  elif [ -d "$PARENT_DIR" ]; then
+    folder="$PARENT_DIR"
+  elif [ -d "$(echo "$DIR" | cut -d "/" -f2)" ]; then
+    folder="$(echo "$DIR" | cut -d "/" -f2)"
+  else
+    echo "We are assuming a root drive install for space-checking purposes."
+    folder='/'
+  fi
+
+  local FREESPACEINKB
+  FREESPACEINKB="$(df -Pk "$folder" | sed 1d | grep -v used | awk '{ print $4 "\t" }')"
+  echo "Detected available space in Kb: $FREESPACEINKB" >&5
+  local FREESPACEINGB
+  FREESPACEINGB=$((FREESPACEINKB / 1024 / 1024))
+  echo "$FREESPACEINGB"
+}
+
+isContainerOrPod() {
+  local cgroup=/proc/1/cgroup
+  test -f $cgroup && (grep -qE ':cpuset:/(docker|kubepods)' $cgroup || grep -q ':/docker/' $cgroup)
+}
+
+isDockerBuildkit() {
+  local cgroup=/proc/1/cgroup
+  test -f $cgroup && grep -q ':cpuset:/docker/buildkit' $cgroup
+}
+
+isDockerContainer() {
+  [ -e /.dockerenv ]
+}
+
+inDocker() {
+  if isContainerOrPod || isDockerBuildkit || isDockerContainer; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# Start OS-specific detection and work
+if [[ "$OSTYPE" == "lin"* ]]; then
+  # Check if root or sudo
+  root=false
+  if [ "$EUID" = 0 ]; then
+    root=true
+  elif command -v id >/dev/null && [ "$(id -u)" = 0 ]; then
+    root=true
+  elif [ "$UID" = 0 ]; then
+    root=true
+  fi
+
+  check_storage_space
+  update_kohya_ss
+
+  distro=get_distro_name
+  family=get_distro_family
+  echo "Raw detected distro string: $distro" >&4
+  echo "Raw detected distro family string: $family" >&4
+
+  if "$distro" | grep -qi "Ubuntu" || "$family" | grep -qi "Ubuntu"; then
+    echo "Ubuntu detected."
+    if [ $(dpkg-query -W -f='${Status}' python3-tk 2>/dev/null | grep -c "ok installed") = 0 ]; then
+      # if [ "$root" = true ]; then
+        echo "This script needs YOU to install the missing python3-tk packages. Please install with:"
+        echo " "
+        if [ "$RUNPOD" = true ]; then
+          bash apt update -y && apt install -y python3-tk
+        else
+          echo "sudo apt update -y && sudo apt install -y python3-tk"
+        fi
+        exit 1
+      # else
+      #   echo "This script needs to be run as root or via sudo to install packages."
+      #   exit 1
+      # fi
+    else
+      echo "Python TK found..."
+    fi
+  elif "$distro" | grep -Eqi "Fedora|CentOS|Redhat"; then
+    echo "Redhat or Redhat base detected."
+    if ! rpm -qa | grep -qi python3-tkinter; then
+      # if [ "$root" = true ]; then
+        echo "This script needs you to install the missing python3-tk packages. Please install with:\n\n"
+        echo "sudo dnf install python3-tkinter -y >&3"
+        exit 1
+      # else
+      #   echo "This script needs to be run as root or via sudo to install packages."
+      #   exit 1
+      # fi
+    else
+      echo "Python TK found..."
+    fi
+  elif "$distro" | grep -Eqi "arch" || "$family" | grep -qi "arch"; then
+    echo "Arch Linux or Arch base detected."
+    if ! pacman -Qi tk >/dev/null; then
+      # if [ "$root" = true ]; then
+        echo "This script needs you to install the missing python3-tk packages. Please install with:\n\n"
+        echo "pacman --noconfirm -S tk >&3"
+        exit 1
+      # else
+      #   echo "This script needs to be run as root or via sudo to install packages."
+      #   exit 1
+      # fi
+    else
+      echo "Python TK found..."
+    fi
+  elif "$distro" | grep -Eqi "opensuse" || "$family" | grep -qi "opensuse"; then
+    echo "OpenSUSE detected."
+    if ! rpm -qa | grep -qi python-tk; then
+      # if [ "$root" = true ]; then
+        echo "This script needs you to install the missing python3-tk packages. Please install with:\n\n"
+        echo "zypper install -y python-tk >&3"
+        exit 1
+      # else
+      #   echo "This script needs to be run as root or via sudo to install packages."
+      #   exit 1
+      # fi
+    else
+      echo "Python TK found..."
+    fi
+  elif [ "$distro" = "None" ] || [ "$family" = "None" ]; then
+    if [ "$distro" = "None" ]; then
+      echo "We could not detect your distribution of Linux. Please file a bug report on github with the contents of your /etc/os-release file."
+    fi
+
+    if [ "$family" = "None" ]; then
+      echo "We could not detect the family of your Linux distribution. Please file a bug report on github with the contents of your /etc/os-release file."
+    fi
+  fi
+
+  install_python_dependencies
+
+  # We need just a little bit more setup for non-interactive environments
+  if [ "$RUNPOD" = true ]; then
+    if inDocker; then
+      # We get the site-packages from python itself, then cut the string, so no other code changes required.
+      VENV_DIR=$(python -c "import site; print(site.getsitepackages()[0])")
+      VENV_DIR="${VENV_DIR%/lib/python3.10/site-packages}"
+    fi
+
+    # Symlink paths
+    libnvinfer_plugin_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.7"
+    libnvinfer_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.7"
+    libcudart_symlink="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.11.0"
+
+    #Target file paths
+    libnvinfer_plugin_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.8"
+    libnvinfer_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.8"
+    libcudart_target="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.12"
+
+    # echo "Checking symlinks now."
+    # create_symlinks "$libnvinfer_plugin_symlink" "$libnvinfer_plugin_target"
+    # create_symlinks "$libnvinfer_symlink" "$libnvinfer_target"
+    # create_symlinks "$libcudart_symlink" "$libcudart_target"
+
+    # if [ -d "${VENV_DIR}/lib/python3.10/site-packages/tensorrt/" ]; then
+    #   export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${VENV_DIR}/lib/python3.10/site-packages/tensorrt/"
+    # else
+    #   echo "${VENV_DIR}/lib/python3.10/site-packages/tensorrt/ not found; not linking library."
+    # fi
+
+    # if [ -d "${VENV_DIR}/lib/python3.10/site-packages/tensorrt/" ]; then
+    #   export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${VENV_DIR}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/"
+    # else
+    #   echo "${VENV_DIR}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/ not found; not linking library."
+    # fi
+
+    configure_accelerate
+
+    # This is a non-interactive environment, so just directly call gui.sh after all setup steps are complete.
+    if [ "$SKIP_GUI" = false ]; then
+      if command -v bash >/dev/null; then
+        if [ "$PUBLIC" = false ]; then
+          bash "$DIR"/gui.sh --headless
+          exit 0
+        else
+          bash "$DIR"/gui.sh --headless --share
+          exit 0
+        fi
+      else
+        # This shouldn't happen, but we're going to try to help.
+        if [ "$PUBLIC" = false ]; then
+          sh "$DIR"/gui.sh --headless
+          exit 0
+        else
+          sh "$DIR"/gui.sh --headless --share
+          exit 0
+        fi
+      fi
+    fi
+  fi
+
+  echo -e "Setup finished! Run \e[0;92m./gui.sh\e[0m to start."
+  echo "Please note if you'd like to expose your public server you need to run ./gui.sh --share"
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  # The initial setup script to prep the environment on macOS
+  # xformers has been omitted as that is for Nvidia GPUs only
+
+  if ! command -v brew >/dev/null; then
+    echo "Please install homebrew first. This is a requirement for the remaining setup."
+    echo "You can find that here: https://brew.sh"
+    #shellcheck disable=SC2016
+    echo 'The "brew" command should be in $PATH to be detected.'
+    exit 1
+  fi
+
+  check_storage_space
+
+  # Install base python packages
+  echo "Installing Python 3.10 if not found."
+  if ! brew ls --versions python@3.10 >/dev/null; then
+    echo "Installing Python 3.10."
+    brew install python@3.10 >&3
+  else
+    echo "Python 3.10 found!"
+  fi
+  echo "Installing Python-TK 3.10 if not found."
+  if ! brew ls --versions python-tk@3.10 >/dev/null; then
+    echo "Installing Python TK 3.10."
+    brew install python-tk@3.10 >&3
+  else
+    echo "Python Tkinter 3.10 found!"
+  fi
+
+  update_kohya_ss
+
+  if ! install_python_dependencies; then
+    echo "You may need to install Python. The command for this is brew install python@3.10."
+  fi
+
+  configure_accelerate
+  echo -e "Setup finished! Run ./gui.sh to start."
+elif [[ "$OSTYPE" == "cygwin" ]]; then
+  # Cygwin is a standalone suite of Linux utilities on Windows
+  echo "This hasn't been validated on cygwin yet."
+elif [[ "$OSTYPE" == "msys" ]]; then
+  # MinGW has the msys environment which is a standalone suite of Linux utilities on Windows
+  # "git bash" on Windows may also be detected as msys.
+  echo "This hasn't been validated in msys (mingw) on Windows yet."
+fi
diff --git a/setup/check_local_modules.py b/setup/check_local_modules.py
new file mode 100644
index 0000000000000000000000000000000000000000..c809ba67e44b569a90b92297e521119b12137e34
--- /dev/null
+++ b/setup/check_local_modules.py
@@ -0,0 +1,33 @@
+import argparse
+import subprocess
+
+# Define color variables
+yellow_text = "\033[1;33m"
+blue_text = "\033[1;34m"
+reset_text = "\033[0m"
+
+# Parse command line arguments
+parser = argparse.ArgumentParser()
+parser.add_argument('--no_question', action='store_true')
+args = parser.parse_args()
+
+# Run pip freeze and capture the output
+output = subprocess.getoutput("pip freeze")
+
+# Remove lines containing "WARNING"
+output_lines = [line for line in output.splitlines() if "WARNING" not in line]
+
+# Reconstruct the output string without warning lines
+output = "\n".join(output_lines)
+
+# Check if modules are found in the output
+if output:
+    print(f"{yellow_text}=============================================================")
+    print("Modules installed outside the virtual environment were found.")
+    print("This can cause issues. Please review the installed modules.\n")
+    print("You can uninstall all local modules with:\n")
+    print(f"{blue_text}deactivate")
+    print("pip freeze > uninstall.txt")
+    print("pip uninstall -y -r uninstall.txt")
+    print(f"{yellow_text}============================================================={reset_text}")
+    print('')
diff --git a/setup/create_user_files.py b/setup/create_user_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c3d00302dbe407cd1bc5b681cce34253872cd8a
--- /dev/null
+++ b/setup/create_user_files.py
@@ -0,0 +1,37 @@
+import os
+
+bat_content = r'''@echo off
+REM Example of how to start the GUI with custom arguments. In this case how to auto launch the browser:
+REM call gui.bat --inbrowser
+REM
+REM You can add many arguments on the same line
+REM
+call gui.bat --inbrowser
+'''
+
+ps1_content = r'''# Example of how to start the GUI with custom arguments. In this case how to auto launch the browser:
+# .\gui.ps1 --inbrowser
+#
+# You can add many arguments on the same line
+#
+# & .\gui.ps1 --inbrowser --server_port 2345
+
+& .\gui.ps1 --inbrowser
+'''
+
+bat_filename = 'gui-user.bat'
+ps1_filename = 'gui-user.ps1'
+
+if not os.path.exists(bat_filename):
+    with open(bat_filename, 'w') as bat_file:
+        bat_file.write(bat_content)
+    print(f"File created: {bat_filename}")
+else:
+    print(f"File already exists: {bat_filename}")
+
+if not os.path.exists(ps1_filename):
+    with open(ps1_filename, 'w') as ps1_file:
+        ps1_file.write(ps1_content)
+    print(f"File created: {ps1_filename}")
+else:
+    print(f"File already exists: {ps1_filename}")
diff --git a/setup/debug_info.py b/setup/debug_info.py
new file mode 100644
index 0000000000000000000000000000000000000000..a4c26c4a5176a30b1dce27ceedda7b28e98160c5
--- /dev/null
+++ b/setup/debug_info.py
@@ -0,0 +1,56 @@
+import platform
+import subprocess
+import os
+
+# Get system information
+system = platform.system()
+release = platform.release()
+version = platform.version()
+machine = platform.machine()
+processor = platform.processor()
+
+# Print system information
+print("System Information:")
+print(f"System: {system}, Release: {release}, Version: {version}, Machine: {machine}, Processor: {processor}")
+
+# Get Python information
+python_version = platform.python_version()
+python_implementation = platform.python_implementation()
+python_compiler = platform.python_compiler()
+
+# Print Python information
+print("\nPython Information:")
+print(f"Version: {python_version}, Implementation: {python_implementation}, Compiler: {python_compiler}")
+
+# Get virtual environment information
+venv = os.environ.get('VIRTUAL_ENV', None)
+
+# Print virtual environment information
+if venv:
+    print("\nVirtual Environment Information:")
+    print(f"Path: {venv}")
+else:
+    print("\nVirtual Environment Information:")
+    print("Not running inside a virtual environment.")
+
+# Get GPU information (requires nvidia-smi to be installed)
+try:
+    output = subprocess.check_output(['nvidia-smi', '--query-gpu=name,memory.total', '--format=csv'])
+    output = output.decode('utf-8').strip().split('\n')[1:]
+    gpu_info = [line.split(', ') for line in output]
+    gpu_name, gpu_vram = gpu_info[0]
+    gpu_vram = gpu_vram.replace(' MiB', '')
+    gpu_vram_warning = int(gpu_vram) < 8000
+except (subprocess.CalledProcessError, FileNotFoundError):
+    gpu_name, gpu_vram = "N/A", "N/A"
+    gpu_vram_warning = False
+
+# Print GPU information
+print("\nGPU Information:")
+print(f"Name: {gpu_name}, VRAM: {gpu_vram} MiB")
+
+# Print VRAM warning if necessary
+if gpu_vram_warning:
+    print('\033[33mWarning: GPU VRAM is less than 8GB and will likelly result in proper operations.\033[0m')
+
+print(' ')
diff --git a/setup/docker_setup.py b/setup/docker_setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2996b5137e2d726b26023ed4c7ac0dc4010b6e9
--- /dev/null
+++ b/setup/docker_setup.py
@@ -0,0 +1,3 @@
+from setuptools import setup, find_packages
+
+setup(name="library", version="1.0.3", packages=find_packages())
\ No newline at end of file
diff --git a/setup/setup_common.py b/setup/setup_common.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d94ca9f3e4c632c4bb61e47e8f07ed55286ba4f
--- /dev/null
+++ b/setup/setup_common.py
@@ -0,0 +1,474 @@
+import subprocess
+import os
+import re
+import sys
+import filecmp
+import logging
+import shutil
+import sysconfig
+import datetime
+import platform
+import pkg_resources
+
+errors = 0  # Define the 'errors' variable before using it
+log = logging.getLogger('sd')
+
+# setup console and file logging
+def setup_logging(clean=False):
+    #
+    # This function was adapted from code written by vladimandic: https://github.com/vladmandic/automatic/commits/master
+    #
+
+    from rich.theme import Theme
+    from rich.logging import RichHandler
+    from rich.console import Console
+    from rich.pretty import install as pretty_install
+    from rich.traceback import install as traceback_install
+
+    console = Console(
+        log_time=True,
+        log_time_format='%H:%M:%S-%f',
+        theme=Theme(
+            {
+                'traceback.border': 'black',
+                'traceback.border.syntax_error': 'black',
+                'inspect.value.border': 'black',
+            }
+        ),
+    )
+    # logging.getLogger("urllib3").setLevel(logging.ERROR)
+    # logging.getLogger("httpx").setLevel(logging.ERROR)
+
+    current_datetime = datetime.datetime.now()
+    current_datetime_str = current_datetime.strftime('%Y%m%d-%H%M%S')
+    log_file = os.path.join(
+        os.path.dirname(__file__),
+        f'../logs/setup/kohya_ss_gui_{current_datetime_str}.log',
+    )
+
+    # Create directories if they don't exist
+    log_directory = os.path.dirname(log_file)
+    os.makedirs(log_directory, exist_ok=True)
+
+    level = logging.INFO
+    logging.basicConfig(
+        level=logging.ERROR,
+        format='%(asctime)s | %(name)s | %(levelname)s | %(module)s | %(message)s',
+        filename=log_file,
+        filemode='a',
+        encoding='utf-8',
+        force=True,
+    )
+    log.setLevel(
+        logging.DEBUG
+    )   # log to file is always at level debug for facility `sd`
+    pretty_install(console=console)
+    traceback_install(
+        console=console,
+        extra_lines=1,
+        width=console.width,
+        word_wrap=False,
+        indent_guides=False,
+        suppress=[],
+    )
+    rh = RichHandler(
+        show_time=True,
+        omit_repeated_times=False,
+        show_level=True,
+        show_path=False,
+        markup=False,
+        rich_tracebacks=True,
+        log_time_format='%H:%M:%S-%f',
+        level=level,
+        console=console,
+    )
+    rh.set_name(level)
+    while log.hasHandlers() and len(log.handlers) > 0:
+        log.removeHandler(log.handlers[0])
+    log.addHandler(rh)
+
+
+def configure_accelerate(run_accelerate=False):
+    #
+    # This function was taken and adapted from code written by jstayco
+    #
+
+    from pathlib import Path
+
+    def env_var_exists(var_name):
+        return var_name in os.environ and os.environ[var_name] != ''
+
+    log.info('Configuring accelerate...')
+    
+    source_accelerate_config_file = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)),
+        '..',
+        'config_files',
+        'accelerate',
+        'default_config.yaml',
+    )
+
+    if not os.path.exists(source_accelerate_config_file):
+        if run_accelerate:
+            run_cmd('accelerate config')
+        else:
+            log.warning(
+                f'Could not find the accelerate configuration file in {source_accelerate_config_file}. Please configure accelerate manually by runningthe option in the menu.'
+            )
+    
+    log.debug(
+        f'Source accelerate config location: {source_accelerate_config_file}'
+    )
+
+    target_config_location = None
+
+    log.debug(
+        f"Environment variables: HF_HOME: {os.environ.get('HF_HOME')}, "
+        f"LOCALAPPDATA: {os.environ.get('LOCALAPPDATA')}, "
+        f"USERPROFILE: {os.environ.get('USERPROFILE')}"
+    )
+    if env_var_exists('HF_HOME'):
+        target_config_location = Path(
+            os.environ['HF_HOME'], 'accelerate', 'default_config.yaml'
+        )
+    elif env_var_exists('LOCALAPPDATA'):
+        target_config_location = Path(
+            os.environ['LOCALAPPDATA'],
+            'huggingface',
+            'accelerate',
+            'default_config.yaml',
+        )
+    elif env_var_exists('USERPROFILE'):
+        target_config_location = Path(
+            os.environ['USERPROFILE'],
+            '.cache',
+            'huggingface',
+            'accelerate',
+            'default_config.yaml',
+        )
+
+    log.debug(f'Target config location: {target_config_location}')
+
+    if target_config_location:
+        if not target_config_location.is_file():
+            target_config_location.parent.mkdir(parents=True, exist_ok=True)
+            log.debug(
+                f'Target accelerate config location: {target_config_location}'
+            )
+            shutil.copyfile(
+                source_accelerate_config_file, target_config_location
+            )
+            log.info(
+                f'Copied accelerate config file to: {target_config_location}'
+            )
+        else:
+            if run_accelerate:
+                run_cmd('accelerate config')
+            else:
+                log.warning(
+                    'Could not automatically configure accelerate. Please manually configure accelerate with the option in the menu or with: accelerate config.'
+                )
+    else:
+        if run_accelerate:
+            run_cmd('accelerate config')
+        else:
+            log.warning(
+                'Could not automatically configure accelerate. Please manually configure accelerate with the option in the menu or with: accelerate config.'
+            )
+
+
+def check_torch():
+    #
+    # This function was adapted from code written by vladimandic: https://github.com/vladmandic/automatic/commits/master
+    #
+
+    # Check for nVidia toolkit or AMD toolkit
+    if shutil.which('nvidia-smi') is not None or os.path.exists(
+        os.path.join(
+            os.environ.get('SystemRoot') or r'C:\Windows',
+            'System32',
+            'nvidia-smi.exe',
+        )
+    ):
+        log.info('nVidia toolkit detected')
+    elif shutil.which('rocminfo') is not None or os.path.exists(
+        '/opt/rocm/bin/rocminfo'
+    ):
+        log.info('AMD toolkit detected')
+    else:
+        log.info('Using CPU-only Torch')
+
+    try:
+        import torch
+
+        log.info(f'Torch {torch.__version__}')
+
+        # Check if CUDA is available
+        if not torch.cuda.is_available():
+            log.warning('Torch reports CUDA not available')
+        else:
+            if torch.version.cuda:
+                # Log nVidia CUDA and cuDNN versions
+                log.info(
+                    f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}'
+                )
+            elif torch.version.hip:
+                # Log AMD ROCm HIP version
+                log.info(f'Torch backend: AMD ROCm HIP {torch.version.hip}')
+            else:
+                log.warning('Unknown Torch backend')
+
+            # Log information about detected GPUs
+            for device in [
+                torch.cuda.device(i) for i in range(torch.cuda.device_count())
+            ]:
+                log.info(
+                    f'Torch detected GPU: {torch.cuda.get_device_name(device)} VRAM {round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} Arch {torch.cuda.get_device_capability(device)} Cores {torch.cuda.get_device_properties(device).multi_processor_count}'
+                )
+                return int(torch.__version__[0])
+    except Exception as e:
+        # log.warning(f'Could not load torch: {e}')
+        return 0
+
+
+# report current version of code
+def check_repo_version(): # pylint: disable=unused-argument
+    if os.path.exists('.release'):
+        with open(os.path.join('./.release'), 'r', encoding='utf8') as file:
+            release= file.read()
+        
+        log.info(f'Version: {release}')
+    else:
+        log.debug('Could not read release...')
+    
+# execute git command
+def git(arg: str, folder: str = None, ignore: bool = False):
+    #
+    # This function was adapted from code written by vladimandic: https://github.com/vladmandic/automatic/commits/master
+    #
+    
+    git_cmd = os.environ.get('GIT', "git")
+    result = subprocess.run(f'"{git_cmd}" {arg}', check=False, shell=True, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=folder or '.')
+    txt = result.stdout.decode(encoding="utf8", errors="ignore")
+    if len(result.stderr) > 0:
+        txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore")
+    txt = txt.strip()
+    if result.returncode != 0 and not ignore:
+        global errors # pylint: disable=global-statement
+        errors += 1
+        log.error(f'Error running git: {folder} / {arg}')
+        if 'or stash them' in txt:
+            log.error(f'Local changes detected: check log for details...')
+        log.debug(f'Git output: {txt}')
+
+
+def pip(arg: str, ignore: bool = False, quiet: bool = False, show_stdout: bool = False):
+    # arg = arg.replace('>=', '==')
+    if not quiet:
+        log.info(f'Installing package: {arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace("  ", " ").strip()}')
+    log.debug(f"Running pip: {arg}")
+    if show_stdout:
+        subprocess.run(f'"{sys.executable}" -m pip {arg}', shell=True, check=False, env=os.environ)
+    else:
+        result = subprocess.run(f'"{sys.executable}" -m pip {arg}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        txt = result.stdout.decode(encoding="utf8", errors="ignore")
+        if len(result.stderr) > 0:
+            txt += ('\n' if len(txt) > 0 else '') + result.stderr.decode(encoding="utf8", errors="ignore")
+        txt = txt.strip()
+        if result.returncode != 0 and not ignore:
+            global errors # pylint: disable=global-statement
+            errors += 1
+            log.error(f'Error running pip: {arg}')
+            log.debug(f'Pip output: {txt}')
+        return txt
+
+
+def installed(package, friendly: str = None):
+    #
+    # This function was adapted from code written by vladimandic: https://github.com/vladmandic/automatic/commits/master
+    #
+    
+    # Remove brackets and their contents from the line using regular expressions
+    # e.g., diffusers[torch]==0.10.2 becomes diffusers==0.10.2
+    package = re.sub(r'\[.*?\]', '', package)
+
+    try:
+        if friendly:
+            pkgs = friendly.split()
+        else:
+            pkgs = [
+                p
+                for p in package.split()
+                if not p.startswith('-') and not p.startswith('=')
+            ]
+            pkgs = [
+                p.split('/')[-1] for p in pkgs
+            ]   # get only package name if installing from URL
+        
+        for pkg in pkgs:
+            if '>=' in pkg:
+                pkg_name, pkg_version = [x.strip() for x in pkg.split('>=')]
+            elif '==' in pkg:
+                pkg_name, pkg_version = [x.strip() for x in pkg.split('==')]
+            else:
+                pkg_name, pkg_version = pkg.strip(), None
+
+            spec = pkg_resources.working_set.by_key.get(pkg_name, None)
+            if spec is None:
+                spec = pkg_resources.working_set.by_key.get(pkg_name.lower(), None)
+            if spec is None:
+                spec = pkg_resources.working_set.by_key.get(pkg_name.replace('_', '-'), None)
+
+            if spec is not None:
+                version = pkg_resources.get_distribution(pkg_name).version
+                log.debug(f'Package version found: {pkg_name} {version}')
+
+                if pkg_version is not None:
+                    if '>=' in pkg:
+                        ok = version >= pkg_version
+                    else:
+                        ok = version == pkg_version
+
+                    if not ok:
+                        log.warning(f'Package wrong version: {pkg_name} {version} required {pkg_version}')
+                        return False
+            else:
+                log.debug(f'Package version not found: {pkg_name}')
+                return False
+
+        return True
+    except ModuleNotFoundError:
+        log.debug(f'Package not installed: {pkgs}')
+        return False
+
+
+# install package using pip if not already installed
+def install(
+    #
+    # This function was adapted from code written by vladimandic: https://github.com/vladmandic/automatic/commits/master
+    #
+    package,
+    friendly: str = None,
+    ignore: bool = False,
+    reinstall: bool = False,
+    show_stdout: bool = False,
+):
+    # Remove anything after '#' in the package variable
+    package = package.split('#')[0].strip()
+
+    if reinstall:
+        global quick_allowed   # pylint: disable=global-statement
+        quick_allowed = False
+    if reinstall or not installed(package, friendly):
+        pip(f'install --upgrade {package}', ignore=ignore, show_stdout=show_stdout)
+
+
+
+def process_requirements_line(line, show_stdout: bool = False):
+    # Remove brackets and their contents from the line using regular expressions
+    # e.g., diffusers[torch]==0.10.2 becomes diffusers==0.10.2
+    package_name = re.sub(r'\[.*?\]', '', line)
+    install(line, package_name, show_stdout=show_stdout)
+
+
+def install_requirements(requirements_file, check_no_verify_flag=False, show_stdout: bool = False):
+    if check_no_verify_flag:
+        log.info(f'Verifying modules instalation status from {requirements_file}...')
+    else:
+        log.info(f'Installing modules from {requirements_file}...')
+    with open(requirements_file, 'r', encoding='utf8') as f:
+        # Read lines from the requirements file, strip whitespace, and filter out empty lines, comments, and lines starting with '.'
+        if check_no_verify_flag:
+            lines = [
+                line.strip()
+                for line in f.readlines()
+                if line.strip() != ''
+                and not line.startswith('#')
+                and line is not None
+                and 'no_verify' not in line
+            ]
+        else:
+            lines = [
+                line.strip()
+                for line in f.readlines()
+                if line.strip() != ''
+                and not line.startswith('#')
+                and line is not None
+            ]
+
+        # Iterate over each line and install the requirements
+        for line in lines:
+            # Check if the line starts with '-r' to include another requirements file
+            if line.startswith('-r'):
+                # Get the path to the included requirements file
+                included_file = line[2:].strip()
+                # Expand the included requirements file recursively
+                install_requirements(included_file, check_no_verify_flag=check_no_verify_flag, show_stdout=show_stdout)
+            else:
+                process_requirements_line(line, show_stdout=show_stdout)
+
+
+def ensure_base_requirements():
+    try:
+        import rich   # pylint: disable=unused-import
+    except ImportError:
+        install('--upgrade rich', 'rich')
+
+
+def run_cmd(run_cmd):
+    try:
+        subprocess.run(run_cmd, shell=True, check=False, env=os.environ)
+    except subprocess.CalledProcessError as e:
+        print(f'Error occurred while running command: {run_cmd}')
+        print(f'Error: {e}')
+
+
+# check python version
+def check_python(ignore=True, skip_git=False):
+    #
+    # This function was adapted from code written by vladimandic: https://github.com/vladmandic/automatic/commits/master
+    #
+
+    supported_minors = [9, 10]
+    log.info(f'Python {platform.python_version()} on {platform.system()}')
+    if not (
+        int(sys.version_info.major) == 3
+        and int(sys.version_info.minor) in supported_minors
+    ):
+        log.error(
+            f'Incompatible Python version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} required 3.{supported_minors}'
+        )
+        if not ignore:
+            sys.exit(1)
+    if not skip_git:
+        git_cmd = os.environ.get('GIT', 'git')
+        if shutil.which(git_cmd) is None:
+            log.error('Git not found')
+            if not ignore:
+                sys.exit(1)
+    else:
+        git_version = git('--version', folder=None, ignore=False)
+        log.debug(f'Git {git_version.replace("git version", "").strip()}')
+
+
+def delete_file(file_path):
+    if os.path.exists(file_path):
+        os.remove(file_path)
+
+
+def write_to_file(file_path, content):
+    try:
+        with open(file_path, 'w') as file:
+            file.write(content)
+    except IOError as e:
+        print(f'Error occurred while writing to file: {file_path}')
+        print(f'Error: {e}')
+
+
+def clear_screen():
+    # Check the current operating system to execute the correct clear screen command
+    if os.name == 'nt':  # If the operating system is Windows
+        os.system('cls')
+    else:  # If the operating system is Linux or Mac
+        os.system('clear')
+
diff --git a/setup/setup_linux.py b/setup/setup_linux.py
new file mode 100644
index 0000000000000000000000000000000000000000..a69e442229d91a90df857fecb500fbebf1dc7e65
--- /dev/null
+++ b/setup/setup_linux.py
@@ -0,0 +1,37 @@
+import argparse
+import logging
+import setup_common
+
+errors = 0  # Define the 'errors' variable before using it
+log = logging.getLogger('sd')
+
+# ANSI escape code for yellow color
+YELLOW = '\033[93m'
+RESET_COLOR = '\033[0m'
+
+
+def main_menu(platform_requirements_file, show_stdout: bool = False, no_run_accelerate: bool = False):
+    log.info("Installing python dependencies. This could take a few minutes as it downloads files.")
+    log.info("If this operation ever runs too long, you can rerun this script in verbose mode to check.")
+    
+    setup_common.check_repo_version()
+    setup_common.check_python()
+
+    # Upgrade pip if needed
+    setup_common.install('--upgrade pip')
+    setup_common.install_requirements(platform_requirements_file, check_no_verify_flag=False, show_stdout=show_stdout)
+    if not no_run_accelerate:
+        setup_common.configure_accelerate(run_accelerate=False)
+
+
+if __name__ == '__main__':
+    setup_common.ensure_base_requirements()
+    setup_common.setup_logging()
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--platform-requirements-file', dest='platform_requirements_file', default='requirements_linux.txt', help='Path to the platform-specific requirements file')
+    parser.add_argument('--show_stdout', dest='show_stdout', action='store_true', help='Whether to show stdout during installation')
+    parser.add_argument('--no_run_accelerate', dest='no_run_accelerate', action='store_true', help='Whether to not run accelerate config')
+    args = parser.parse_args()
+
+    main_menu(args.platform_requirements_file, show_stdout=args.show_stdout, no_run_accelerate=args.no_run_accelerate)
diff --git a/setup/setup_runpod.py b/setup/setup_runpod.py
new file mode 100644
index 0000000000000000000000000000000000000000..d909191d42f8cda028c583468002787263ea55fc
--- /dev/null
+++ b/setup/setup_runpod.py
@@ -0,0 +1,69 @@
+import argparse
+import logging
+import setup_common
+import os
+import shutil
+
+errors = 0  # Define the 'errors' variable before using it
+log = logging.getLogger('sd')
+
+# ANSI escape code for yellow color
+YELLOW = '\033[93m'
+RESET_COLOR = '\033[0m'
+
+def configure_accelerate():
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    cache_dir = "/root/.cache/huggingface/accelerate"
+    
+    log.info("Configuring accelerate...")
+    os.makedirs(cache_dir, exist_ok=True)
+
+    config_file_src = os.path.join(script_dir, "config_files", "accelerate", "runpod.yaml")
+    config_file_dest = os.path.join(cache_dir, "default_config.yaml")
+    shutil.copyfile(config_file_src, config_file_dest)
+
+
+def setup_environment():
+    # Get the directory the script is run from
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Install tk and python3.10-venv
+    log.info("Install tk and python3.10-venv...")
+    subprocess.run(['apt', 'update', '-y'])
+    subprocess.run(['apt', 'install', '-y', 'python3-tk', 'python3.10-venv'])
+
+    # Check if the venv folder doesn't exist
+    venv_dir = os.path.join(script_dir, 'venv')
+    if not os.path.exists(venv_dir):
+        log.info("Creating venv...")
+        subprocess.run(['python3', '-m', 'venv', venv_dir])
+
+    # Activate the virtual environment
+    log.info("Activate venv...")
+    activate_script = os.path.join(venv_dir, 'bin', 'activate')
+    activate_command = f'source "{activate_script}" || exit 1'
+    subprocess.run(activate_command, shell=True, executable='/bin/bash')
+
+
+def main_menu(platform_requirements_file):
+    log.info("Installing python dependencies. This could take a few minutes as it downloads files.")
+    log.info("If this operation ever runs too long, you can rerun this script in verbose mode to check.")
+
+    setup_common.check_repo_version()
+    setup_common.check_python()
+
+    # Upgrade pip if needed
+    setup_common.install('--upgrade pip')
+    setup_common.install_requirements(platform_requirements_file, check_no_verify_flag=False, show_stdout=True)
+    configure_accelerate()
+
+
+if __name__ == '__main__':
+    setup_common.ensure_base_requirements()
+    setup_common.setup_logging()
+    
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--platform-requirements-file', dest='platform_requirements_file', default='requirements_runpod.txt', help='Path to the platform-specific requirements file')
+    args = parser.parse_args()
+    
+    main_menu(args.platform_requirements_file)
diff --git a/setup/setup_windows.py b/setup/setup_windows.py
new file mode 100644
index 0000000000000000000000000000000000000000..8f2c1720a37544f78d89295b8bc73ad4e57a234d
--- /dev/null
+++ b/setup/setup_windows.py
@@ -0,0 +1,202 @@
+import subprocess
+import os
+import filecmp
+import logging
+import shutil
+import sysconfig
+import setup_common
+
+errors = 0  # Define the 'errors' variable before using it
+log = logging.getLogger('sd')
+
+# ANSI escape code for yellow color
+YELLOW = '\033[93m'
+RESET_COLOR = '\033[0m'
+
+
+def cudann_install():
+    cudnn_src = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), '..\cudnn_windows'
+    )
+    cudnn_dest = os.path.join(sysconfig.get_paths()['purelib'], 'torch', 'lib')
+
+    log.info(f'Checking for CUDNN files in {cudnn_dest}...')
+    if os.path.exists(cudnn_src):
+        if os.path.exists(cudnn_dest):
+            # check for different files
+            filecmp.clear_cache()
+            for file in os.listdir(cudnn_src):
+                src_file = os.path.join(cudnn_src, file)
+                dest_file = os.path.join(cudnn_dest, file)
+                # if dest file exists, check if it's different
+                if os.path.exists(dest_file):
+                    if not filecmp.cmp(src_file, dest_file, shallow=False):
+                        shutil.copy2(src_file, cudnn_dest)
+                else:
+                    shutil.copy2(src_file, cudnn_dest)
+            log.info('Copied CUDNN 8.6 files to destination')
+        else:
+            log.warning(f'Destination directory {cudnn_dest} does not exist')
+    else:
+        log.error(f'Installation Failed: "{cudnn_src}" could not be found.')
+
+
+def sync_bits_and_bytes_files():
+    import filecmp
+
+    """
+    Check for "different" bitsandbytes Files and copy only if necessary.
+    This function is specific for Windows OS.
+    """
+
+    # Only execute on Windows
+    if os.name != 'nt':
+        print('This function is only applicable to Windows OS.')
+        return
+
+    try:
+        log.info(f'Copying bitsandbytes files...')
+        # Define source and destination directories
+        source_dir = os.path.join(os.getcwd(), 'bitsandbytes_windows')
+
+        dest_dir_base = os.path.join(
+            sysconfig.get_paths()['purelib'], 'bitsandbytes'
+        )
+
+        # Clear file comparison cache
+        filecmp.clear_cache()
+
+        # Iterate over each file in source directory
+        for file in os.listdir(source_dir):
+            source_file_path = os.path.join(source_dir, file)
+
+            # Decide the destination directory based on file name
+            if file in ('main.py', 'paths.py'):
+                dest_dir = os.path.join(dest_dir_base, 'cuda_setup')
+            else:
+                dest_dir = dest_dir_base
+
+            dest_file_path = os.path.join(dest_dir, file)
+
+            # Compare the source file with the destination file
+            if os.path.exists(dest_file_path) and filecmp.cmp(
+                source_file_path, dest_file_path
+            ):
+                log.debug(
+                    f'Skipping {source_file_path} as it already exists in {dest_dir}'
+                )
+            else:
+                # Copy file from source to destination, maintaining original file's metadata
+                log.debug(f'Copy {source_file_path} to {dest_dir}')
+                shutil.copy2(source_file_path, dest_dir)
+
+    except FileNotFoundError as fnf_error:
+        log.error(f'File not found error: {fnf_error}')
+    except PermissionError as perm_error:
+        log.error(f'Permission error: {perm_error}')
+    except Exception as e:
+        log.error(f'An unexpected error occurred: {e}')
+
+
+def install_kohya_ss_torch1():
+    setup_common.check_repo_version()
+    setup_common.check_python()
+
+    # Upgrade pip if needed
+    setup_common.install('--upgrade pip')
+
+    if setup_common.check_torch() == 2:
+        input(
+            f'{YELLOW}\nTorch 2 is already installed in the venv. To install Torch 1 delete the venv and re-run setup.bat\n\nHit enter to continue...{RESET_COLOR}'
+        )
+        return
+
+    # setup_common.install(
+    #     'torch==1.12.1+cu116 torchvision==0.13.1+cu116 --index-url https://download.pytorch.org/whl/cu116',
+    #     'torch torchvision'
+    # )
+    # setup_common.install(
+    #     'https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl -U -I --no-deps',
+    #     'xformers-0.0.14'
+    # )
+    setup_common.install_requirements('requirements_windows_torch1.txt', check_no_verify_flag=False)
+    sync_bits_and_bytes_files()
+    setup_common.configure_accelerate(run_accelerate=True)
+    # run_cmd(f'accelerate config')
+
+
+def install_kohya_ss_torch2():
+    setup_common.check_repo_version()
+    setup_common.check_python()
+
+    # Upgrade pip if needed
+    setup_common.install('--upgrade pip')
+
+    if setup_common.check_torch() == 1:
+        input(
+            f'{YELLOW}\nTorch 1 is already installed in the venv. To install Torch 2 delete the venv and re-run setup.bat\n\nHit any key to acknowledge.{RESET_COLOR}'
+        )
+        return
+
+    # setup_common.install(
+    #     'torch==2.0.1+cu118 torchvision==0.15.2+cu118 --index-url https://download.pytorch.org/whl/cu118',
+    #     'torch torchvision'
+    # )
+    setup_common.install_requirements('requirements_windows_torch2.txt', check_no_verify_flag=False)
+    # install('https://huggingface.co/r4ziel/xformers_pre_built/resolve/main/triton-2.0.0-cp310-cp310-win_amd64.whl', 'triton', reinstall=reinstall)
+    sync_bits_and_bytes_files()
+    setup_common.configure_accelerate(run_accelerate=True)
+    # run_cmd(f'accelerate config')
+
+
+def main_menu():
+    setup_common.clear_screen()
+    while True:
+        print('\nKohya_ss GUI setup menu:\n')
+        print('1. Install kohya_ss gui')
+        print('2. (Optional) Install cudann files')
+        print('3. (Optional) Install bitsandbytes-windows')
+        print('4. (Optional) Manually configure accelerate')
+        print('5. (Optional) Start Kohya_ss GUI in browser')
+        print('6. Quit')
+
+        choice = input('\nEnter your choice: ')
+        print('')
+
+        if choice == '1':
+            while True:
+                print('1. Torch 1 (legacy)')
+                print('2. Torch 2 (recommended)')
+                print('3. Cancel')
+                choice_torch = input('\nEnter your choice: ')
+                print('')
+
+                if choice_torch == '1':
+                    install_kohya_ss_torch1()
+                    break
+                elif choice_torch == '2':
+                    install_kohya_ss_torch2()
+                    break
+                elif choice_torch == '3':
+                    break
+                else:
+                    print('Invalid choice. Please enter a number between 1-3.')
+        elif choice == '2':
+            cudann_install()
+        elif choice == '3':
+            setup_common.install('--upgrade bitsandbytes-windows', reinstall=True)
+        elif choice == '4':
+            setup_common.run_cmd('accelerate config')
+        elif choice == '5':
+            subprocess.Popen('start cmd /k .\gui.bat --inbrowser', shell=True) # /k keep the terminal open on quit. /c would close the terminal instead
+        elif choice == '6':
+            print('Quitting the program.')
+            break
+        else:
+            print('Invalid choice. Please enter a number between 1-5.')
+
+
+if __name__ == '__main__':
+    setup_common.ensure_base_requirements()
+    setup_common.setup_logging()
+    main_menu()
diff --git a/setup/update_bitsandbytes.py b/setup/update_bitsandbytes.py
new file mode 100644
index 0000000000000000000000000000000000000000..ee8b2ae609f53080e0e80dd8122908236437e21f
--- /dev/null
+++ b/setup/update_bitsandbytes.py
@@ -0,0 +1,49 @@
+import os
+import sysconfig
+import filecmp
+import shutil
+
+def sync_bits_and_bytes_files():
+    """
+    Check for "different" bitsandbytes Files and copy only if necessary.
+    This function is specific for Windows OS.
+    """
+    
+    # Only execute on Windows
+    if os.name != "nt":
+        print("This function is only applicable to Windows OS.")
+        return
+
+    try:
+        # Define source and destination directories
+        source_dir = os.path.join(os.getcwd(), "bitsandbytes_windows")
+
+        dest_dir_base = os.path.join(sysconfig.get_paths()["purelib"], "bitsandbytes")
+        
+        # Clear file comparison cache
+        filecmp.clear_cache()
+        
+        # Iterate over each file in source directory
+        for file in os.listdir(source_dir):
+            source_file_path = os.path.join(source_dir, file)
+
+            # Decide the destination directory based on file name
+            if file in ("main.py", "paths.py"):
+                dest_dir = os.path.join(dest_dir_base, "cuda_setup")
+            else:
+                dest_dir = dest_dir_base
+
+            # Copy file from source to destination, maintaining original file's metadata
+            print(f'Copy {source_file_path} to {dest_dir}')
+            shutil.copy2(source_file_path, dest_dir)
+
+    except FileNotFoundError as fnf_error:
+        print(f"File not found error: {fnf_error}")
+    except PermissionError as perm_error:
+        print(f"Permission error: {perm_error}")
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+
+
+if __name__ == "__main__":
+    sync_bits_and_bytes_files()
\ No newline at end of file
diff --git a/setup/validate_requirements.py b/setup/validate_requirements.py
new file mode 100644
index 0000000000000000000000000000000000000000..73e94eb6509bd7803fb4bf0ee510e9423d398174
--- /dev/null
+++ b/setup/validate_requirements.py
@@ -0,0 +1,101 @@
+import os
+import re
+import sys
+import shutil
+import argparse
+import setup_common
+
+# Get the absolute path of the current file's directory (Kohua_SS project directory)
+project_directory = os.path.dirname(os.path.abspath(__file__))
+
+# Check if the "setup" directory is present in the project_directory
+if "setup" in project_directory:
+    # If the "setup" directory is present, move one level up to the parent directory
+    project_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# Add the project directory to the beginning of the Python search path
+sys.path.insert(0, project_directory)
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+def check_torch():
+    # Check for nVidia toolkit or AMD toolkit
+    if shutil.which('nvidia-smi') is not None or os.path.exists(
+        os.path.join(
+            os.environ.get('SystemRoot') or r'C:\Windows',
+            'System32',
+            'nvidia-smi.exe',
+        )
+    ):
+        log.info('nVidia toolkit detected')
+    elif shutil.which('rocminfo') is not None or os.path.exists(
+        '/opt/rocm/bin/rocminfo'
+    ):
+        log.info('AMD toolkit detected')
+    else:
+        log.info('Using CPU-only Torch')
+
+    try:
+        import torch
+
+        log.info(f'Torch {torch.__version__}')
+
+        # Check if CUDA is available
+        if not torch.cuda.is_available():
+            log.warning('Torch reports CUDA not available')
+        else:
+            if torch.version.cuda:
+                # Log nVidia CUDA and cuDNN versions
+                log.info(
+                    f'Torch backend: nVidia CUDA {torch.version.cuda} cuDNN {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else "N/A"}'
+                )
+            elif torch.version.hip:
+                # Log AMD ROCm HIP version
+                log.info(f'Torch backend: AMD ROCm HIP {torch.version.hip}')
+            else:
+                log.warning('Unknown Torch backend')
+
+            # Log information about detected GPUs
+            for device in [
+                torch.cuda.device(i) for i in range(torch.cuda.device_count())
+            ]:
+                log.info(
+                    f'Torch detected GPU: {torch.cuda.get_device_name(device)} VRAM {round(torch.cuda.get_device_properties(device).total_memory / 1024 / 1024)} Arch {torch.cuda.get_device_capability(device)} Cores {torch.cuda.get_device_properties(device).multi_processor_count}'
+                )
+                return int(torch.__version__[0])
+    except Exception as e:
+        log.error(f'Could not load torch: {e}')
+        sys.exit(1)
+
+
+def main():
+    setup_common.check_repo_version()
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(
+        description='Validate that requirements are satisfied.'
+    )
+    parser.add_argument(
+        '-r',
+        '--requirements',
+        type=str,
+        help='Path to the requirements file.',
+    )
+    parser.add_argument('--debug', action='store_true', help='Debug on')
+    args = parser.parse_args()
+
+    torch_ver = check_torch()
+    
+    if args.requirements:
+        setup_common.install_requirements(args.requirements, check_no_verify_flag=True)
+    else:
+        if torch_ver == 1:
+            setup_common.install_requirements('requirements_windows_torch1.txt', check_no_verify_flag=True)
+        else:
+            setup_common.install_requirements('requirements_windows_torch2.txt', check_no_verify_flag=True)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/style.css b/style.css
new file mode 100644
index 0000000000000000000000000000000000000000..78d34a9479035a9963f34d3e5f3c9499e1ac8592
--- /dev/null
+++ b/style.css
@@ -0,0 +1,34 @@
+#open_folder_small{
+    height: auto;
+    min-width: auto;
+    flex-grow: 0;
+    padding-left: 0.25em;
+    padding-right: 0.25em;
+}
+
+#open_folder{
+    height: auto;
+    flex-grow: 0;
+    padding-left: 0.25em;
+    padding-right: 0.25em;
+}
+
+#number_input{
+    min-width: min-content;
+    flex-grow: 0.3;
+    padding-left: 0.75em;
+    padding-right: 0.75em;
+}
+
+.ver-class {
+    color: #808080;
+    font-size: small;
+    text-align: right;
+    padding-right: 1em;
+}
+
+#myDropdown {
+    height: auto;
+    width: 33%;
+    flex-grow: 0;
+}
\ No newline at end of file
diff --git a/test/config/SDXL-Standard-Adafactor.json b/test/config/SDXL-Standard-Adafactor.json
new file mode 100644
index 0000000000000000000000000000000000000000..d06e9193836efe2faf2bb9d720b230bea9b1aae1
--- /dev/null
+++ b/test/config/SDXL-Standard-Adafactor.json
@@ -0,0 +1,106 @@
+{
+  "LoRA_type": "Standard",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": false,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": true,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0,
+  "caption_extension": ".none-use-foldername",
+  "clip_skip": "1",
+  "color_aug": false,
+  "conv_alpha": 64,
+  "conv_alphas": "",
+  "conv_dim": 64,
+  "conv_dims": "",
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 4,
+  "factor": -1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": true,
+  "keep_tokens": "0",
+  "learning_rate": 0.001,
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "constant",
+  "lr_scheduler_num_cycles": "1",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "1024,1024",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_snr_gamma": 10,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "custom",
+  "module_dropout": 0,
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "network_alpha": 64,
+  "network_dim": 64,
+  "network_dropout": 0,
+  "no_token_padding": false,
+  "noise_offset": 0.0357,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "SDXL-Standard-AdamW8bit",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "D:/models/sdxl/sd_xl_base_0.9-pruned.safetensors",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 0,
+  "sample_prompts": "",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 0,
+  "sdxl": true,
+  "sdxl_cache_text_encoder_outputs": true,
+  "sdxl_no_half_vae": true,
+  "seed": "",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 0.0,
+  "train_batch_size": 1,
+  "train_data_dir": "./test/img",
+  "train_on_input": true,
+  "training_comment": "",
+  "unet_lr": 0.001,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": false,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/SDXL-Standard-AdamW.json b/test/config/SDXL-Standard-AdamW.json
new file mode 100644
index 0000000000000000000000000000000000000000..cbc48cc544a10c45f5a77b0e559707497d5f5981
--- /dev/null
+++ b/test/config/SDXL-Standard-AdamW.json
@@ -0,0 +1,106 @@
+{
+  "LoRA_type": "Standard",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": false,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": true,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0,
+  "caption_extension": ".none-use-foldername",
+  "clip_skip": "1",
+  "color_aug": false,
+  "conv_alpha": 64,
+  "conv_alphas": "",
+  "conv_dim": 64,
+  "conv_dims": "",
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 1,
+  "factor": -1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": true,
+  "keep_tokens": "0",
+  "learning_rate": 0.0001,
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "cosine",
+  "lr_scheduler_num_cycles": "1",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "1024,1024",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_snr_gamma": 10,
+  "min_timestep": 0,
+  "mixed_precision": "fp16",
+  "model_list": "custom",
+  "module_dropout": 0,
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "network_alpha": 64,
+  "network_dim": 64,
+  "network_dropout": 0,
+  "no_token_padding": false,
+  "noise_offset": 0.0357,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "SDXL-Standard-AdamW",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "D:/models/sdxl/sd_xl_base_0.9-pruned.safetensors",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 0,
+  "sample_prompts": "",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 0,
+  "sdxl": true,
+  "sdxl_cache_text_encoder_outputs": true,
+  "sdxl_no_half_vae": true,
+  "seed": "",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 0.0,
+  "train_batch_size": 1,
+  "train_data_dir": "./test/img",
+  "train_on_input": true,
+  "training_comment": "",
+  "unet_lr": 0.0001,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": false,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/SDXL-Standard-AdamW8bit.json b/test/config/SDXL-Standard-AdamW8bit.json
new file mode 100644
index 0000000000000000000000000000000000000000..499d52bb4c5026a870d5a1b511882136a97620d9
--- /dev/null
+++ b/test/config/SDXL-Standard-AdamW8bit.json
@@ -0,0 +1,106 @@
+{
+  "LoRA_type": "Standard",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": false,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": true,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0,
+  "caption_extension": ".none-use-foldername",
+  "clip_skip": "1",
+  "color_aug": false,
+  "conv_alpha": 64,
+  "conv_alphas": "",
+  "conv_dim": 64,
+  "conv_dims": "",
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 1,
+  "factor": -1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": true,
+  "keep_tokens": "0",
+  "learning_rate": 1e-06,
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "cosine",
+  "lr_scheduler_num_cycles": "1",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "1024,1024",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_snr_gamma": 10,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "custom",
+  "module_dropout": 0,
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "network_alpha": 64,
+  "network_dim": 64,
+  "network_dropout": 0,
+  "no_token_padding": false,
+  "noise_offset": 0.0357,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "SDXL-Standard-AdamW8bit",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "D:/models/sdxl/sd_xl_base_0.9-pruned.safetensors",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 0,
+  "sample_prompts": "",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 0,
+  "sdxl": true,
+  "sdxl_cache_text_encoder_outputs": true,
+  "sdxl_no_half_vae": true,
+  "seed": "",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 0.0,
+  "train_batch_size": 1,
+  "train_data_dir": "./test/img",
+  "train_on_input": true,
+  "training_comment": "",
+  "unet_lr": 1e-06,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": false,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/Standard-AdamW.json b/test/config/Standard-AdamW.json
new file mode 100644
index 0000000000000000000000000000000000000000..64c5dc4f339ae209c5d92db63d8c37f2f52cbede
--- /dev/null
+++ b/test/config/Standard-AdamW.json
@@ -0,0 +1,104 @@
+{
+  "LoRA_type": "Standard",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "conv_alpha": 8,
+  "conv_alphas": "",
+  "conv_dim": 16,
+  "conv_dims": "",
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 4,
+  "factor": -1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 0.0001,
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "cosine",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_snr_gamma": 10,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "module_dropout": 0.1,
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "network_alpha": 16,
+  "network_dim": 16,
+  "network_dropout": 0.1,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Multires",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "Standard-Adamw",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0.1,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 20,
+  "sample_prompts": "a painting of man wearing a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 1,
+  "sdxl": false,
+  "sdxl_cache_text_encoder_outputs": false,
+  "sdxl_no_half_vae": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 0.0001,
+  "train_batch_size": 4,
+  "train_data_dir": "./test/img",
+  "train_on_input": false,
+  "training_comment": "",
+  "unet_lr": 0.0001,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": true,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/dreambooth-Adafactor.json b/test/config/dreambooth-Adafactor.json
new file mode 100644
index 0000000000000000000000000000000000000000..f61b309cbeee3dbc77a63c997496781436bfcf8b
--- /dev/null
+++ b/test/config/dreambooth-Adafactor.json
@@ -0,0 +1,76 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 1,
+  "color_aug": false,
+  "enable_bucket": true,
+  "epoch": 1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 4.0,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 0.0001,
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "constant",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "min_snr_gamma": 0,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "Adafactor",
+  "optimizer_args": "scale_parameter=False relative_step=False warmup_init=False",
+  "output_dir": "./test/output",
+  "output_name": "dreambooth-Adafactor",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "sdxl": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "train_batch_size": 1,
+  "train_data_dir": "./test/img",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/dreambooth-AdamW.json b/test/config/dreambooth-AdamW.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f41f7d369928969c9ed982fedcebfd3b7e32dd7
--- /dev/null
+++ b/test/config/dreambooth-AdamW.json
@@ -0,0 +1,73 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 64,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "enable_bucket": true,
+  "epoch": 1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 5e-05,
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "constant",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "min_snr_gamma": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "db-AdamW",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "train_batch_size": 4,
+  "train_data_dir": "./test/img",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/dreambooth-AdamW8bit.json b/test/config/dreambooth-AdamW8bit.json
new file mode 100644
index 0000000000000000000000000000000000000000..09865620fc0a4fe47d86cf8762a9cc385031b647
--- /dev/null
+++ b/test/config/dreambooth-AdamW8bit.json
@@ -0,0 +1,73 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 64,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "enable_bucket": true,
+  "epoch": 1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 5e-05,
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "constant",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "min_snr_gamma": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "db-AdamW8bit",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "train_batch_size": 4,
+  "train_data_dir": "./test/img",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/dreambooth-DAdaptAdam.json b/test/config/dreambooth-DAdaptAdam.json
new file mode 100644
index 0000000000000000000000000000000000000000..de04bef1dc335ebfd4daca93d36a854519251695
--- /dev/null
+++ b/test/config/dreambooth-DAdaptAdam.json
@@ -0,0 +1,73 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "enable_bucket": true,
+  "epoch": 1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 1.0,
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "cosine",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "min_snr_gamma": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Multires",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "DAdaptAdam",
+  "optimizer_args": "decouple=True weight_decay=0.6 betas=0.9,0.99 use_bias_correction=True",
+  "output_dir": "./test/output",
+  "output_name": "db-DAdaptAdam",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "train_batch_size": 1,
+  "train_data_dir": "./test/img",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/dreambooth-Prodigy.json b/test/config/dreambooth-Prodigy.json
new file mode 100644
index 0000000000000000000000000000000000000000..0bbc337aeba30a137513ab6d815de85058a72d53
--- /dev/null
+++ b/test/config/dreambooth-Prodigy.json
@@ -0,0 +1,73 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "enable_bucket": true,
+  "epoch": 1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 1.0,
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "cosine",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "min_snr_gamma": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Multires",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "Prodigy",
+  "optimizer_args": "decouple=True weight_decay=0.6 betas=0.9,0.99 use_bias_correction=True",
+  "output_dir": "./test/output",
+  "output_name": "db-Prodigy",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "train_batch_size": 1,
+  "train_data_dir": "./test/img",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/dreambooth.json b/test/config/dreambooth.json
new file mode 100644
index 0000000000000000000000000000000000000000..57e32c64f68e454e7b372ca282cd67aad1244ada
--- /dev/null
+++ b/test/config/dreambooth.json
@@ -0,0 +1,76 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 1,
+  "color_aug": false,
+  "enable_bucket": true,
+  "epoch": 1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 4.0,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 1.0,
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "constant",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "min_snr_gamma": 0,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "db",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "sdxl": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "train_batch_size": 1,
+  "train_data_dir": "./test/img",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/finetune-AdamW.json b/test/config/finetune-AdamW.json
new file mode 100644
index 0000000000000000000000000000000000000000..d3128ae8293dc1178db7f71f39897fe7de758497
--- /dev/null
+++ b/test/config/finetune-AdamW.json
@@ -0,0 +1,82 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "batch_size": "8",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0,
+  "caption_extension": ".txt",
+  "caption_metadata_filename": "meta-1_cap.json",
+  "clip_skip": 1,
+  "color_aug": false,
+  "create_buckets": false,
+  "create_caption": true,
+  "dataset_repeats": "50",
+  "epoch": 2,
+  "flip_aug": false,
+  "full_fp16": false,
+  "full_path": true,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": false,
+  "image_folder": ".\\test\\img\\10_darius kawasaki person",
+  "keep_tokens": 0,
+  "latent_metadata_filename": "meta-1_lat.json",
+  "learning_rate": 1e-05,
+  "logging_dir": "./test/ft",
+  "lr_scheduler": "cosine_with_restarts",
+  "lr_warmup": 10,
+  "max_bucket_reso": "1024",
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "min_bucket_reso": "256",
+  "min_snr_gamma": 0,
+  "mixed_precision": "bf16",
+  "model_list": "stabilityai/stable-diffusion-xl-base-0.9",
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "noise_offset": 0,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "test_ft",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "stabilityai/stable-diffusion-xl-base-0.9",
+  "random_crop": false,
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 0,
+  "sample_prompts": "",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "bf16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "sdxl_cache_text_encoder_outputs": false,
+  "sdxl_checkbox": true,
+  "sdxl_no_half_vae": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "train_batch_size": 4,
+  "train_dir": "./test",
+  "train_text_encoder": true,
+  "use_latent_files": "No",
+  "use_wandb": false,
+  "v2": true,
+  "v_parameterization": true,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/iA3-Prodigy.json b/test/config/iA3-Prodigy.json
new file mode 100644
index 0000000000000000000000000000000000000000..23f3bd78313900af147992d6064323172c9dd6ac
--- /dev/null
+++ b/test/config/iA3-Prodigy.json
@@ -0,0 +1,104 @@
+{
+  "LoRA_type": "LyCORIS/iA3",
+  "adaptive_noise_scale": 0.005,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.5,
+  "caption_extension": ".txt",
+  "clip_skip": 2,
+  "color_aug": false,
+  "conv_alpha": 8,
+  "conv_alphas": "",
+  "conv_dim": 16,
+  "conv_dims": "",
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 4,
+  "factor": -1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "keep_tokens": 1,
+  "learning_rate": 1.0,
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "cosine",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 8,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_snr_gamma": 5,
+  "mixed_precision": "bf16",
+  "model_list": "custom",
+  "module_dropout": 0.1,
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "network_alpha": 1024,
+  "network_dim": 1024,
+  "network_dropout": 0.3,
+  "no_token_padding": false,
+  "noise_offset": 0.05,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "Prodigy",
+  "optimizer_args": "d_coef=1.0 weight_decay=0.01 safeguard_warmup=True use_bias_correction=False",
+  "output_dir": "./test/output",
+  "output_name": "iA3-Prodigy",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0.1,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 1,
+  "sample_every_n_steps": 0,
+  "sample_prompts": "a man wearing a gas mask, by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 1,
+  "sdxl": false,
+  "sdxl_cache_text_encoder_outputs": false,
+  "sdxl_no_half_vae": false,
+  "seed": "31337",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 1.0,
+  "train_batch_size": 1,
+  "train_data_dir": "./test/img",
+  "train_on_input": false,
+  "training_comment": "rentry.co/ProdiAgy",
+  "unet_lr": 1.0,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": true,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/locon-AdamW.json b/test/config/locon-AdamW.json
new file mode 100644
index 0000000000000000000000000000000000000000..6c38f8e16eaea17224d3839b84eae3c2294ce175
--- /dev/null
+++ b/test/config/locon-AdamW.json
@@ -0,0 +1,106 @@
+{
+  "LoRA_type": "Kohya LoCon",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 64,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "conv_alpha": 8,
+  "conv_alphas": "",
+  "conv_dim": 16,
+  "conv_dims": "",
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 1,
+  "factor": -1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 4,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 0.0005,
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "constant",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_snr_gamma": 0,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "module_dropout": 0.1,
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "network_alpha": 8,
+  "network_dim": 16,
+  "network_dropout": 0.1,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "locon-AdamW",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0.1,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 1,
+  "sdxl": false,
+  "sdxl_cache_text_encoder_outputs": false,
+  "sdxl_no_half_vae": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 0.0001,
+  "train_batch_size": 1,
+  "train_data_dir": "./test/img",
+  "train_on_input": false,
+  "training_comment": "",
+  "unet_lr": 0.0001,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": false,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/locon-AdamW8bit.json b/test/config/locon-AdamW8bit.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1666ef3789dd233515dfbe1b5e6c7b9f3ec959
--- /dev/null
+++ b/test/config/locon-AdamW8bit.json
@@ -0,0 +1,101 @@
+{
+  "LoRA_type": "Kohya LoCon",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 64,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "conv_alpha": 8,
+  "conv_alphas": "",
+  "conv_dim": 16,
+  "conv_dims": "",
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 1,
+  "factor": -1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 0.0005,
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "constant",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_snr_gamma": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "module_dropout": 0.1,
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "network_alpha": 8,
+  "network_dim": 16,
+  "network_dropout": 0.1,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "locon-AdamW8bit",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0.1,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 1,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 0.0001,
+  "train_batch_size": 4,
+  "train_data_dir": "./test/img",
+  "train_on_input": false,
+  "training_comment": "",
+  "unet_lr": 0.0001,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": false,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/locon-Prodigy.json b/test/config/locon-Prodigy.json
new file mode 100644
index 0000000000000000000000000000000000000000..cf88d25204ba2c010ae1d99b49114775b39d4eee
--- /dev/null
+++ b/test/config/locon-Prodigy.json
@@ -0,0 +1,101 @@
+{
+  "LoRA_type": "Kohya LoCon",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "conv_alpha": 8,
+  "conv_alphas": "",
+  "conv_dim": 16,
+  "conv_dims": "",
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 1,
+  "factor": -1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 1.0,
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "cosine",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_snr_gamma": 10,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "module_dropout": 0.1,
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "network_alpha": 8,
+  "network_dim": 16,
+  "network_dropout": 0.1,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Multires",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "Prodigy",
+  "optimizer_args": "decouple=True weight_decay=0.6 betas=0.9,0.99 use_bias_correction=True",
+  "output_dir": "./test/output",
+  "output_name": "locon-Prodigy",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0.1,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 1,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 1.0,
+  "train_batch_size": 4,
+  "train_data_dir": "./test/img",
+  "train_on_input": false,
+  "training_comment": "",
+  "unet_lr": 1.0,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": true,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/config/loha-Prodigy.json b/test/config/loha-Prodigy.json
new file mode 100644
index 0000000000000000000000000000000000000000..9ccbf4827e61019a596bb4840ac5953bccc04902
--- /dev/null
+++ b/test/config/loha-Prodigy.json
@@ -0,0 +1,106 @@
+{
+  "LoRA_type": "LyCORIS/LoHa",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "--log_prefix=xl-loha",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": false,
+  "bucket_reso_steps": 64,
+  "cache_latents": true,
+  "cache_latents_to_disk": true,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0,
+  "caption_extension": ".txt",
+  "clip_skip": "1",
+  "color_aug": false,
+  "conv_alpha": 4,
+  "conv_alphas": "",
+  "conv_dim": 4,
+  "conv_dims": "",
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 10,
+  "factor": -1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": true,
+  "keep_tokens": "0",
+  "learning_rate": 0.002,
+  "logging_dir": "E:\\froggy\\loha\\logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "cosine",
+  "lr_scheduler_num_cycles": "1",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "1024,1024",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_snr_gamma": 5,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "custom",
+  "module_dropout": 0,
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "network_alpha": 8,
+  "network_dim": 8,
+  "network_dropout": 0,
+  "no_token_padding": false,
+  "noise_offset": 0.0357,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "weight_decay=0.05 betas=0.9,0.98",
+  "output_dir": "d:\\lycoris\\sdxl",
+  "output_name": "froddy-loha-sx_v1.0a",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "D:/models/sdxl/sd_xl_base_0.9.safetensors",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 0,
+  "sample_prompts": "",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "bf16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 0,
+  "sdxl": true,
+  "sdxl_cache_text_encoder_outputs": true,
+  "sdxl_no_half_vae": true,
+  "seed": "17415",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 0.0,
+  "train_batch_size": 1,
+  "train_data_dir": "E:\\froggy\\img",
+  "train_on_input": false,
+  "training_comment": "",
+  "unet_lr": 0.002,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": false,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}
\ No newline at end of file
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ccfc199477904f7a9362827cbae450c845963159
Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki.jpg differ
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b589f7c7e3509c5854b856f8bce4021d2b32828d
--- /dev/null
+++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki.txt	
@@ -0,0 +1 @@
+a painting of a steam punk skull with a gas mask , by darius kawasaki
\ No newline at end of file
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..bb062cacace77dabdc92a73876afddc1e42fdfb4
Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.jpg differ
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e7ded5fa8c1c6bb2aa71e0da722f1c30c270bc4e
--- /dev/null
+++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_2.txt	
@@ -0,0 +1 @@
+a painting of a man with a skull on his head , by darius kawasaki
\ No newline at end of file
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e84720572d71bbdb49401a33cb00683377da75bc
Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.jpg differ
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4ef297cf866fe3b6cab53410c8697a5be49a30a2
--- /dev/null
+++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_3.txt	
@@ -0,0 +1 @@
+a painting of a woman with a helmet on her head , by darius kawasaki
\ No newline at end of file
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..8821fa28253106d946ab2f35010e85976c2aa895
Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.jpg differ
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4abb67e8f8186eb214bd3ef5c4a2b8212d5bc2a2
--- /dev/null
+++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_4.txt	
@@ -0,0 +1 @@
+a painting of a horned man with a goat head , by darius kawasaki
\ No newline at end of file
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..590a097a61140ed39fc50060c6b000e9fcc1e4ff
Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.jpg differ
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.txt
new file mode 100644
index 0000000000000000000000000000000000000000..406814f85c1da098a95bc9a6da32e469e180006f
--- /dev/null
+++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_5.txt	
@@ -0,0 +1 @@
+a painting of a man playing a piano , by darius kawasaki
\ No newline at end of file
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..4a39ce4289b5da8d635a2d90a8be1edeef18b73b
Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.jpg differ
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.txt
new file mode 100644
index 0000000000000000000000000000000000000000..55de4a66eb345b65f0fb2fa8b4856c3939b5ba55
--- /dev/null
+++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_6.txt	
@@ -0,0 +1 @@
+a painting of a robot sitting on a rock , by darius kawasaki
\ No newline at end of file
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0c1f50f42b6243a5a170ca0c2c3068c7897d29f2
Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.jpg differ
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e36dca828d342e326158f04bb5f4c5784df2df7e
--- /dev/null
+++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_7.txt	
@@ -0,0 +1 @@
+a painting of a soldier with a helmet on , by darius kawasaki
\ No newline at end of file
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.jpg b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7d3380be2126a32df205c8dbfb06dfbde1e78354
Binary files /dev/null and b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.jpg differ
diff --git a/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.txt b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0087fedfc24537feea1f3d26bc7426dc32b388b8
--- /dev/null
+++ b/test/img/10_darius kawasaki person/Dariusz_Zawadzki_8.txt	
@@ -0,0 +1 @@
+a painting of a giant crab with a large body , by darius kawasaki
\ No newline at end of file
diff --git a/textual_inversion_gui.py b/textual_inversion_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..702241ce6e4a16449b82720fccd4f43a85c6b705
--- /dev/null
+++ b/textual_inversion_gui.py
@@ -0,0 +1,1022 @@
+# v1: initial release
+# v2: add open and save folder icons
+# v3: Add new Utilities tab for Dreambooth folder preparation
+# v3.1: Adding captionning of images to utilities
+
+import gradio as gr
+import json
+import math
+import os
+import subprocess
+import pathlib
+import argparse
+from datetime import datetime
+from library.common_gui import (
+    get_file_path,
+    get_saveasfile_path,
+    color_aug_changed,
+    save_inference_file,
+    run_cmd_advanced_training,
+    run_cmd_training,
+    update_my_data,
+    check_if_model_exist,
+    output_message,
+    verify_image_folder_pattern,
+    SaveConfigFile,
+    save_to_file
+)
+from library.class_configuration_file import ConfigurationFile
+from library.class_source_model import SourceModel
+from library.class_basic_training import BasicTraining
+from library.class_advanced_training import AdvancedTraining
+from library.class_folders import Folders
+from library.class_sdxl_parameters import SDXLParameters
+from library.tensorboard_gui import (
+    gradio_tensorboard,
+    start_tensorboard,
+    stop_tensorboard,
+)
+from library.dreambooth_folder_creation_gui import (
+    gradio_dreambooth_folder_creation_tab,
+)
+from library.utilities import utilities_tab
+from library.class_sample_images import SampleImages, run_cmd_sample
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+def save_configuration(
+    save_as,
+    file_path,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl,
+    logging_dir,
+    train_data_dir,
+    reg_data_dir,
+    output_dir,
+    max_resolution,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    cache_latents,
+    cache_latents_to_disk,
+    caption_extension,
+    enable_bucket,
+    gradient_checkpointing,
+    full_fp16,
+    no_token_padding,
+    stop_text_encoder_training,
+    # use_8bit_adam,
+    xformers,
+    save_model_as,
+    shuffle_caption,
+    save_state,
+    resume,
+    prior_loss_weight,
+    color_aug,
+    flip_aug,
+    clip_skip,
+    vae,
+    output_name,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    mem_eff_attn,
+    gradient_accumulation_steps,
+    model_list,
+    token_string,
+    init_word,
+    num_vectors_per_token,
+    max_train_steps,
+    weights,
+    template,
+    keep_tokens,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    min_timestep,
+    max_timestep,
+    sdxl_no_half_vae
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+
+    original_file_path = file_path
+
+    save_as_bool = True if save_as.get('label') == 'True' else False
+
+    if save_as_bool:
+        log.info('Save as...')
+        file_path = get_saveasfile_path(file_path)
+    else:
+        log.info('Save...')
+        if file_path == None or file_path == '':
+            file_path = get_saveasfile_path(file_path)
+
+    # log.info(file_path)
+
+    if file_path == None or file_path == '':
+        return original_file_path  # In case a file_path was provided and the user decide to cancel the open action
+
+    # Extract the destination directory from the file path
+    destination_directory = os.path.dirname(file_path)
+
+    # Create the destination directory if it doesn't exist
+    if not os.path.exists(destination_directory):
+        os.makedirs(destination_directory)
+
+    SaveConfigFile(parameters=parameters, file_path=file_path, exclusion=['file_path', 'save_as'])
+
+    return file_path
+
+
+def open_configuration(
+    ask_for_file,
+    file_path,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl,
+    logging_dir,
+    train_data_dir,
+    reg_data_dir,
+    output_dir,
+    max_resolution,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    cache_latents,
+    cache_latents_to_disk,
+    caption_extension,
+    enable_bucket,
+    gradient_checkpointing,
+    full_fp16,
+    no_token_padding,
+    stop_text_encoder_training,
+    # use_8bit_adam,
+    xformers,
+    save_model_as,
+    shuffle_caption,
+    save_state,
+    resume,
+    prior_loss_weight,
+    color_aug,
+    flip_aug,
+    clip_skip,
+    vae,
+    output_name,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    mem_eff_attn,
+    gradient_accumulation_steps,
+    model_list,
+    token_string,
+    init_word,
+    num_vectors_per_token,
+    max_train_steps,
+    weights,
+    template,
+    keep_tokens,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    min_timestep,
+    max_timestep,
+    sdxl_no_half_vae
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+
+    ask_for_file = True if ask_for_file.get('label') == 'True' else False
+
+    original_file_path = file_path
+
+    if ask_for_file:
+        file_path = get_file_path(file_path)
+
+    if not file_path == '' and not file_path == None:
+        # load variables from JSON file
+        with open(file_path, 'r') as f:
+            my_data = json.load(f)
+            log.info('Loading config...')
+            # Update values to fix deprecated use_8bit_adam checkbox and set appropriate optimizer if it is set to True
+            my_data = update_my_data(my_data)
+    else:
+        file_path = original_file_path  # In case a file_path was provided and the user decide to cancel the open action
+        my_data = {}
+
+    values = [file_path]
+    for key, value in parameters:
+        # Set the value in the dictionary to the corresponding value in `my_data`, or the default value if not found
+        if not key in ['ask_for_file', 'file_path']:
+            values.append(my_data.get(key, value))
+    return tuple(values)
+
+
+def train_model(
+    headless,
+    print_only,
+    pretrained_model_name_or_path,
+    v2,
+    v_parameterization,
+    sdxl,
+    logging_dir,
+    train_data_dir,
+    reg_data_dir,
+    output_dir,
+    max_resolution,
+    learning_rate,
+    lr_scheduler,
+    lr_warmup,
+    train_batch_size,
+    epoch,
+    save_every_n_epochs,
+    mixed_precision,
+    save_precision,
+    seed,
+    num_cpu_threads_per_process,
+    cache_latents,
+    cache_latents_to_disk,
+    caption_extension,
+    enable_bucket,
+    gradient_checkpointing,
+    full_fp16,
+    no_token_padding,
+    stop_text_encoder_training_pct,
+    # use_8bit_adam,
+    xformers,
+    save_model_as,
+    shuffle_caption,
+    save_state,
+    resume,
+    prior_loss_weight,
+    color_aug,
+    flip_aug,
+    clip_skip,
+    vae,
+    output_name,
+    max_token_length,
+    max_train_epochs,
+    max_data_loader_n_workers,
+    mem_eff_attn,
+    gradient_accumulation_steps,
+    model_list,  # Keep this. Yes, it is unused here but required given the common list used
+    token_string,
+    init_word,
+    num_vectors_per_token,
+    max_train_steps,
+    weights,
+    template,
+    keep_tokens,
+    persistent_data_loader_workers,
+    bucket_no_upscale,
+    random_crop,
+    bucket_reso_steps,
+    caption_dropout_every_n_epochs,
+    caption_dropout_rate,
+    optimizer,
+    optimizer_args,
+    noise_offset_type,
+    noise_offset,
+    adaptive_noise_scale,
+    multires_noise_iterations,
+    multires_noise_discount,
+    sample_every_n_steps,
+    sample_every_n_epochs,
+    sample_sampler,
+    sample_prompts,
+    additional_parameters,
+    vae_batch_size,
+    min_snr_gamma,
+    save_every_n_steps,
+    save_last_n_steps,
+    save_last_n_steps_state,
+    use_wandb,
+    wandb_api_key,
+    scale_v_pred_loss_like_noise_pred,
+    min_timestep,
+    max_timestep,
+    sdxl_no_half_vae
+):
+    # Get list of function parameters and values
+    parameters = list(locals().items())
+    
+    print_only_bool = True if print_only.get('label') == 'True' else False
+    log.info(f'Start training TI...')
+
+    headless_bool = True if headless.get('label') == 'True' else False
+
+    if pretrained_model_name_or_path == '':
+        output_message(
+            msg='Source model information is missing', headless=headless_bool
+        )
+        return
+
+    if train_data_dir == '':
+        output_message(
+            msg='Image folder path is missing', headless=headless_bool
+        )
+        return
+
+    if not os.path.exists(train_data_dir):
+        output_message(
+            msg='Image folder does not exist', headless=headless_bool
+        )
+        return
+
+    if not verify_image_folder_pattern(train_data_dir):
+        return
+
+    if reg_data_dir != '':
+        if not os.path.exists(reg_data_dir):
+            output_message(
+                msg='Regularisation folder does not exist',
+                headless=headless_bool,
+            )
+            return
+
+        if not verify_image_folder_pattern(reg_data_dir):
+            return
+
+    if output_dir == '':
+        output_message(
+            msg='Output folder path is missing', headless=headless_bool
+        )
+        return
+
+    if token_string == '':
+        output_message(msg='Token string is missing', headless=headless_bool)
+        return
+
+    if init_word == '':
+        output_message(msg='Init word is missing', headless=headless_bool)
+        return
+
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    if check_if_model_exist(
+        output_name, output_dir, save_model_as, headless_bool
+    ):
+        return
+
+    # if float(noise_offset) > 0 and (
+    #     multires_noise_iterations > 0 or multires_noise_discount > 0
+    # ):
+    #     output_message(
+    #         msg="noise offset and multires_noise can't be set at the same time. Only use one or the other.",
+    #         title='Error',
+    #         headless=headless_bool,
+    #     )
+    #     return
+
+    # if optimizer == 'Adafactor' and lr_warmup != '0':
+    #     output_message(
+    #         msg="Warning: lr_scheduler is set to 'Adafactor', so 'LR warmup (% of steps)' will be considered 0.",
+    #         title='Warning',
+    #         headless=headless_bool,
+    #     )
+    #     lr_warmup = '0'
+
+    # Get a list of all subfolders in train_data_dir
+    subfolders = [
+        f
+        for f in os.listdir(train_data_dir)
+        if os.path.isdir(os.path.join(train_data_dir, f))
+    ]
+
+    total_steps = 0
+
+    # Loop through each subfolder and extract the number of repeats
+    for folder in subfolders:
+        # Extract the number of repeats from the folder name
+        repeats = int(folder.split('_')[0])
+
+        # Count the number of images in the folder
+        num_images = len(
+            [
+                f
+                for f, lower_f in (
+                    (file, file.lower())
+                    for file in os.listdir(
+                        os.path.join(train_data_dir, folder)
+                    )
+                )
+                if lower_f.endswith(('.jpg', '.jpeg', '.png', '.webp'))
+            ]
+        )
+
+        # Calculate the total number of steps for this folder
+        steps = repeats * num_images
+        total_steps += steps
+
+        # Print the result
+        log.info(f'Folder {folder}: {steps} steps')
+
+    # Print the result
+    # log.info(f"{total_steps} total steps")
+
+    if reg_data_dir == '':
+        reg_factor = 1
+    else:
+        log.info(
+            'Regularisation images are used... Will double the number of steps required...'
+        )
+        reg_factor = 2
+
+    # calculate max_train_steps
+    if max_train_steps == '':
+        max_train_steps = int(
+            math.ceil(
+                float(total_steps)
+                / int(train_batch_size)
+                / int(gradient_accumulation_steps)
+                * int(epoch)
+                * int(reg_factor)
+            )
+        )
+    else:
+        max_train_steps = int(max_train_steps)
+
+    log.info(f'max_train_steps = {max_train_steps}')
+
+    # calculate stop encoder training
+    if stop_text_encoder_training_pct == None:
+        stop_text_encoder_training = 0
+    else:
+        stop_text_encoder_training = math.ceil(
+            float(max_train_steps) / 100 * int(stop_text_encoder_training_pct)
+        )
+    log.info(f'stop_text_encoder_training = {stop_text_encoder_training}')
+
+    lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+    log.info(f'lr_warmup_steps = {lr_warmup_steps}')
+
+    run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process}'
+    if sdxl:
+        run_cmd += f' "./sdxl_train_textual_inversion.py"'
+    else:
+        run_cmd += f' "./train_textual_inversion.py"'
+        
+    if v2:
+        run_cmd += ' --v2'
+    if v_parameterization:
+        run_cmd += ' --v_parameterization'
+    if enable_bucket:
+        run_cmd += ' --enable_bucket'
+    if no_token_padding:
+        run_cmd += ' --no_token_padding'
+    run_cmd += (
+        f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"'
+    )
+    run_cmd += f' --train_data_dir="{train_data_dir}"'
+    if len(reg_data_dir):
+        run_cmd += f' --reg_data_dir="{reg_data_dir}"'
+    run_cmd += f' --resolution="{max_resolution}"'
+    run_cmd += f' --output_dir="{output_dir}"'
+    if not logging_dir == '':
+        run_cmd += f' --logging_dir="{logging_dir}"'
+    if not stop_text_encoder_training == 0:
+        run_cmd += (
+            f' --stop_text_encoder_training={stop_text_encoder_training}'
+        )
+    if not save_model_as == 'same as source model':
+        run_cmd += f' --save_model_as={save_model_as}'
+    # if not resume == '':
+    #     run_cmd += f' --resume={resume}'
+    if not float(prior_loss_weight) == 1.0:
+        run_cmd += f' --prior_loss_weight={prior_loss_weight}'
+    if not vae == '':
+        run_cmd += f' --vae="{vae}"'
+    if not output_name == '':
+        run_cmd += f' --output_name="{output_name}"'
+    if int(max_token_length) > 75:
+        run_cmd += f' --max_token_length={max_token_length}'
+    if not max_train_epochs == '':
+        run_cmd += f' --max_train_epochs="{max_train_epochs}"'
+    if not max_data_loader_n_workers == '':
+        run_cmd += (
+            f' --max_data_loader_n_workers="{max_data_loader_n_workers}"'
+        )
+    if int(gradient_accumulation_steps) > 1:
+        run_cmd += f' --gradient_accumulation_steps={int(gradient_accumulation_steps)}'
+    
+    if sdxl_no_half_vae:
+        run_cmd += f' --no_half_vae'
+
+    run_cmd += run_cmd_training(
+        learning_rate=learning_rate,
+        lr_scheduler=lr_scheduler,
+        lr_warmup_steps=lr_warmup_steps,
+        train_batch_size=train_batch_size,
+        max_train_steps=max_train_steps,
+        save_every_n_epochs=save_every_n_epochs,
+        mixed_precision=mixed_precision,
+        save_precision=save_precision,
+        seed=seed,
+        caption_extension=caption_extension,
+        cache_latents=cache_latents,
+        cache_latents_to_disk=cache_latents_to_disk,
+        optimizer=optimizer,
+        optimizer_args=optimizer_args,
+    )
+
+    run_cmd += run_cmd_advanced_training(
+        max_train_epochs=max_train_epochs,
+        max_data_loader_n_workers=max_data_loader_n_workers,
+        max_token_length=max_token_length,
+        resume=resume,
+        save_state=save_state,
+        mem_eff_attn=mem_eff_attn,
+        clip_skip=clip_skip,
+        flip_aug=flip_aug,
+        color_aug=color_aug,
+        shuffle_caption=shuffle_caption,
+        gradient_checkpointing=gradient_checkpointing,
+        full_fp16=full_fp16,
+        xformers=xformers,
+        # use_8bit_adam=use_8bit_adam,
+        keep_tokens=keep_tokens,
+        persistent_data_loader_workers=persistent_data_loader_workers,
+        bucket_no_upscale=bucket_no_upscale,
+        random_crop=random_crop,
+        bucket_reso_steps=bucket_reso_steps,
+        caption_dropout_every_n_epochs=caption_dropout_every_n_epochs,
+        caption_dropout_rate=caption_dropout_rate,
+        noise_offset_type=noise_offset_type,
+        noise_offset=noise_offset,
+        adaptive_noise_scale=adaptive_noise_scale,
+        multires_noise_iterations=multires_noise_iterations,
+        multires_noise_discount=multires_noise_discount,
+        additional_parameters=additional_parameters,
+        vae_batch_size=vae_batch_size,
+        min_snr_gamma=min_snr_gamma,
+        save_every_n_steps=save_every_n_steps,
+        save_last_n_steps=save_last_n_steps,
+        save_last_n_steps_state=save_last_n_steps_state,
+        use_wandb=use_wandb,
+        wandb_api_key=wandb_api_key,
+        scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred,
+        min_timestep=min_timestep,
+        max_timestep=max_timestep,
+    )
+    run_cmd += f' --token_string="{token_string}"'
+    run_cmd += f' --init_word="{init_word}"'
+    run_cmd += f' --num_vectors_per_token={num_vectors_per_token}'
+    if not weights == '':
+        run_cmd += f' --weights="{weights}"'
+    if template == 'object template':
+        run_cmd += f' --use_object_template'
+    elif template == 'style template':
+        run_cmd += f' --use_style_template'
+
+    run_cmd += run_cmd_sample(
+        sample_every_n_steps,
+        sample_every_n_epochs,
+        sample_sampler,
+        sample_prompts,
+        output_dir,
+    )
+
+    if print_only_bool:
+        log.warning(
+            'Here is the trainer command as a reference. It will not be executed:\n'
+        )
+        print(run_cmd)
+        
+        save_to_file(run_cmd)
+    else:
+        # Saving config file for model
+        current_datetime = datetime.now()
+        formatted_datetime = current_datetime.strftime("%Y%m%d-%H%M%S")
+        file_path = os.path.join(output_dir, f'{output_name}_{formatted_datetime}.json')
+        
+        log.info(f'Saving training config to {file_path}...')
+
+        SaveConfigFile(parameters=parameters, file_path=file_path, exclusion=['file_path', 'save_as', 'headless', 'print_only'])
+        
+        log.info(run_cmd)
+
+        # Run the command
+        if os.name == 'posix':
+            os.system(run_cmd)
+        else:
+            subprocess.run(run_cmd)
+
+        # check if output_dir/last is a folder... therefore it is a diffuser model
+        last_dir = pathlib.Path(f'{output_dir}/{output_name}')
+
+        if not last_dir.is_dir():
+            # Copy inference model for v2 if required
+            save_inference_file(
+                output_dir, v2, v_parameterization, output_name
+            )
+
+
+def ti_tab(
+    headless=False,
+):
+    dummy_db_true = gr.Label(value=True, visible=False)
+    dummy_db_false = gr.Label(value=False, visible=False)
+    dummy_headless = gr.Label(value=headless, visible=False)
+    
+    with gr.Tab('Training'):
+        gr.Markdown('Train a TI using kohya textual inversion python code...')
+        
+        # Setup Configuration Files Gradio
+        config = ConfigurationFile(headless)
+
+        source_model = SourceModel(
+            save_model_as_choices=[
+                'ckpt',
+                'safetensors',
+            ],
+            headless=headless,
+        )
+
+        with gr.Tab('Folders'):
+            folders = Folders(headless=headless)
+        with gr.Tab('Parameters'):
+            with gr.Row():
+                weights = gr.Textbox(
+                    label='Resume TI training',
+                    placeholder='(Optional) Path to existing TI embeding file to keep training',
+                )
+                weights_file_input = gr.Button(
+                    '📂', elem_id='open_folder_small', visible=(not headless)
+                )
+                weights_file_input.click(
+                    get_file_path,
+                    outputs=weights,
+                    show_progress=False,
+                )
+            with gr.Row():
+                token_string = gr.Textbox(
+                    label='Token string',
+                    placeholder='eg: cat',
+                )
+                init_word = gr.Textbox(
+                    label='Init word',
+                    value='*',
+                )
+                num_vectors_per_token = gr.Slider(
+                    minimum=1,
+                    maximum=75,
+                    value=1,
+                    step=1,
+                    label='Vectors',
+                )
+                max_train_steps = gr.Textbox(
+                    label='Max train steps',
+                    placeholder='(Optional) Maximum number of steps',
+                )
+                template = gr.Dropdown(
+                    label='Template',
+                    choices=[
+                        'caption',
+                        'object template',
+                        'style template',
+                    ],
+                    value='caption',
+                )
+            basic_training = BasicTraining(
+                learning_rate_value='1e-5',
+                lr_scheduler_value='cosine',
+                lr_warmup_value='10',
+            )
+                    
+            # Add SDXL Parameters
+            sdxl_params = SDXLParameters(source_model.sdxl_checkbox, show_sdxl_cache_text_encoder_outputs=False)
+                
+            with gr.Accordion('Advanced Configuration', open=False):
+                advanced_training = AdvancedTraining(headless=headless)
+                advanced_training.color_aug.change(
+                    color_aug_changed,
+                    inputs=[advanced_training.color_aug],
+                    outputs=[basic_training.cache_latents],
+                )
+
+            sample = SampleImages()
+
+        with gr.Tab('Tools'):
+            gr.Markdown(
+                'This section provide Dreambooth tools to help setup your dataset...'
+            )
+            gradio_dreambooth_folder_creation_tab(
+                train_data_dir_input=folders.train_data_dir,
+                reg_data_dir_input=folders.reg_data_dir,
+                output_dir_input=folders.output_dir,
+                logging_dir_input=folders.logging_dir,
+                headless=headless,
+            )
+
+        button_run = gr.Button('Train model', variant='primary')
+
+        button_print = gr.Button('Print training command')
+
+        # Setup gradio tensorboard buttons
+        button_start_tensorboard, button_stop_tensorboard = gradio_tensorboard()
+
+        button_start_tensorboard.click(
+            start_tensorboard,
+            inputs=folders.logging_dir,
+            show_progress=False,
+        )
+
+        button_stop_tensorboard.click(
+            stop_tensorboard,
+            show_progress=False,
+        )
+
+        settings_list = [
+            source_model.pretrained_model_name_or_path,
+            source_model.v2,
+            source_model.v_parameterization,
+            source_model.sdxl_checkbox,
+            folders.logging_dir,
+            folders.train_data_dir,
+            folders.reg_data_dir,
+            folders.output_dir,
+            basic_training.max_resolution,
+            basic_training.learning_rate,
+            basic_training.lr_scheduler,
+            basic_training.lr_warmup,
+            basic_training.train_batch_size,
+            basic_training.epoch,
+            basic_training.save_every_n_epochs,
+            basic_training.mixed_precision,
+            basic_training.save_precision,
+            basic_training.seed,
+            basic_training.num_cpu_threads_per_process,
+            basic_training.cache_latents,
+            basic_training.cache_latents_to_disk,
+            basic_training.caption_extension,
+            basic_training.enable_bucket,
+            advanced_training.gradient_checkpointing,
+            advanced_training.full_fp16,
+            advanced_training.no_token_padding,
+            basic_training.stop_text_encoder_training,
+            advanced_training.xformers,
+            source_model.save_model_as,
+            advanced_training.shuffle_caption,
+            advanced_training.save_state,
+            advanced_training.resume,
+            advanced_training.prior_loss_weight,
+            advanced_training.color_aug,
+            advanced_training.flip_aug,
+            advanced_training.clip_skip,
+            advanced_training.vae,
+            folders.output_name,
+            advanced_training.max_token_length,
+            advanced_training.max_train_epochs,
+            advanced_training.max_data_loader_n_workers,
+            advanced_training.mem_eff_attn,
+            advanced_training.gradient_accumulation_steps,
+            source_model.model_list,
+            token_string,
+            init_word,
+            num_vectors_per_token,
+            max_train_steps,
+            weights,
+            template,
+            advanced_training.keep_tokens,
+            advanced_training.persistent_data_loader_workers,
+            advanced_training.bucket_no_upscale,
+            advanced_training.random_crop,
+            advanced_training.bucket_reso_steps,
+            advanced_training.caption_dropout_every_n_epochs,
+            advanced_training.caption_dropout_rate,
+            basic_training.optimizer,
+            basic_training.optimizer_args,
+            advanced_training.noise_offset_type,
+            advanced_training.noise_offset,
+            advanced_training.adaptive_noise_scale,
+            advanced_training.multires_noise_iterations,
+            advanced_training.multires_noise_discount,
+            sample.sample_every_n_steps,
+            sample.sample_every_n_epochs,
+            sample.sample_sampler,
+            sample.sample_prompts,
+            advanced_training.additional_parameters,
+            advanced_training.vae_batch_size,
+            advanced_training.min_snr_gamma,
+            advanced_training.save_every_n_steps,
+            advanced_training.save_last_n_steps,
+            advanced_training.save_last_n_steps_state,
+            advanced_training.use_wandb,
+            advanced_training.wandb_api_key,
+            advanced_training.scale_v_pred_loss_like_noise_pred,
+            advanced_training.min_timestep,
+            advanced_training.max_timestep,
+            sdxl_params.sdxl_no_half_vae,
+        ]
+
+        config.button_open_config.click(
+            open_configuration,
+            inputs=[dummy_db_true, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name] + settings_list,
+            show_progress=False,
+        )
+
+        config.button_load_config.click(
+            open_configuration,
+            inputs=[dummy_db_false, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name] + settings_list,
+            show_progress=False,
+        )
+
+        config.button_save_config.click(
+            save_configuration,
+            inputs=[dummy_db_false, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name],
+            show_progress=False,
+        )
+
+        config.button_save_as_config.click(
+            save_configuration,
+            inputs=[dummy_db_true, config.config_file_name] + settings_list,
+            outputs=[config.config_file_name],
+            show_progress=False,
+        )
+
+        button_run.click(
+            train_model,
+            inputs=[dummy_headless] + [dummy_db_false] + settings_list,
+            show_progress=False,
+        )
+
+        button_print.click(
+            train_model,
+            inputs=[dummy_headless] + [dummy_db_true] + settings_list,
+            show_progress=False,
+        )
+
+        return (
+            folders.train_data_dir,
+            folders.reg_data_dir,
+            folders.output_dir,
+            folders.logging_dir,
+        )
+
+
+def UI(**kwargs):
+    css = ''
+
+    headless = kwargs.get('headless', False)
+    log.info(f'headless: {headless}')
+
+    if os.path.exists('./style.css'):
+        with open(os.path.join('./style.css'), 'r', encoding='utf8') as file:
+            log.info('Load CSS...')
+            css += file.read() + '\n'
+
+    interface = gr.Blocks(
+        css=css, title='Kohya_ss GUI', theme=gr.themes.Default()
+    )
+
+    with interface:
+        with gr.Tab('Dreambooth TI'):
+            (
+                train_data_dir_input,
+                reg_data_dir_input,
+                output_dir_input,
+                logging_dir_input,
+            ) = ti_tab(headless=headless)
+        with gr.Tab('Utilities'):
+            utilities_tab(
+                train_data_dir_input=train_data_dir_input,
+                reg_data_dir_input=reg_data_dir_input,
+                output_dir_input=output_dir_input,
+                logging_dir_input=logging_dir_input,
+                enable_copy_info_button=True,
+                headless=headless,
+            )
+
+    # Show the interface
+    launch_kwargs = {}
+    username = kwargs.get('username')
+    password = kwargs.get('password')
+    server_port = kwargs.get('server_port', 0)
+    inbrowser = kwargs.get('inbrowser', False)
+    share = kwargs.get('share', False)
+    server_name = kwargs.get('listen')
+
+    launch_kwargs['server_name'] = server_name
+    if username and password:
+        launch_kwargs['auth'] = (username, password)
+    if server_port > 0:
+        launch_kwargs['server_port'] = server_port
+    if inbrowser:
+        launch_kwargs['inbrowser'] = inbrowser
+    if share:
+        launch_kwargs['share'] = share
+    interface.launch(**launch_kwargs)
+
+
+if __name__ == '__main__':
+    # torch.cuda.set_per_process_memory_fraction(0.48)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--listen',
+        type=str,
+        default='127.0.0.1',
+        help='IP to listen on for connections to Gradio',
+    )
+    parser.add_argument(
+        '--username', type=str, default='', help='Username for authentication'
+    )
+    parser.add_argument(
+        '--password', type=str, default='', help='Password for authentication'
+    )
+    parser.add_argument(
+        '--server_port',
+        type=int,
+        default=0,
+        help='Port to run the server listener on',
+    )
+    parser.add_argument(
+        '--inbrowser', action='store_true', help='Open in browser'
+    )
+    parser.add_argument(
+        '--share', action='store_true', help='Share the gradio UI'
+    )
+    parser.add_argument(
+        '--headless', action='store_true', help='Is the server headless'
+    )
+
+    args = parser.parse_args()
+
+    UI(
+        username=args.username,
+        password=args.password,
+        inbrowser=args.inbrowser,
+        server_port=args.server_port,
+        share=args.share,
+        listen=args.listen,
+        headless=args.headless,
+    )
diff --git a/tools/canny.py b/tools/canny.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e0806898786e5251d2e715e33896bb4958a35e8
--- /dev/null
+++ b/tools/canny.py
@@ -0,0 +1,30 @@
+import argparse
+import cv2
+
+
+def canny(args):
+  img = cv2.imread(args.input)
+  img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+
+  canny_img = cv2.Canny(img, args.thres1, args.thres2)
+  # canny_img = 255 - canny_img
+
+  cv2.imwrite(args.output, canny_img)
+  print("done!")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--input", type=str, default=None, help="input path")
+  parser.add_argument("--output", type=str, default=None, help="output path")
+  parser.add_argument("--thres1", type=int, default=32, help="thres1")
+  parser.add_argument("--thres2", type=int, default=224, help="thres2")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+  canny(args)
diff --git a/tools/caption.py b/tools/caption.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd9dd53a966786cadc372a2577c4102aa231aa8b
--- /dev/null
+++ b/tools/caption.py
@@ -0,0 +1,69 @@
+# This script will create the caption text files in the specified folder using the specified file pattern and caption text.
+#
+# eg: python caption.py D:\some\folder\location "*.png, *.jpg, *.webp" "some caption text"
+
+import argparse
+# import glob
+# import os
+from pathlib import Path
+
+def create_caption_files(image_folder: str, file_pattern: str, caption_text: str, caption_file_ext: str, overwrite: bool):
+    # Split the file patterns string and strip whitespace from each pattern
+    patterns = [pattern.strip() for pattern in file_pattern.split(",")]
+
+    # Create a Path object for the image folder
+    folder = Path(image_folder)
+
+    # Iterate over the file patterns
+    for pattern in patterns:
+        # Use the glob method to match the file patterns
+        files = folder.glob(pattern)
+
+        # Iterate over the matched files
+        for file in files:
+            # Check if a text file with the same name as the current file exists in the folder
+            txt_file = file.with_suffix(caption_file_ext)
+            if not txt_file.exists() or overwrite:
+                # Create a text file with the caption text in the folder, if it does not already exist
+                # or if the overwrite argument is True
+                with open(txt_file, "w") as f:
+                    f.write(caption_text)
+
+def main():
+    # Define command-line arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("image_folder", type=str, help="the folder where the image files are located")
+    parser.add_argument("--file_pattern", type=str, default="*.png, *.jpg, *.jpeg, *.webp", help="the pattern to match the image file names")
+    parser.add_argument("--caption_file_ext", type=str, default=".caption", help="the caption file extension.")
+    parser.add_argument("--overwrite", action="store_true", default=False, help="whether to overwrite existing caption files")
+
+    # Create a mutually exclusive group for the caption_text and caption_file arguments
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("--caption_text", type=str, help="the text to include in the caption files")
+    group.add_argument("--caption_file", type=argparse.FileType("r"), help="the file containing the text to include in the caption files")
+
+    # Parse the command-line arguments
+    args = parser.parse_args()
+    image_folder = args.image_folder
+    file_pattern = args.file_pattern
+    caption_file_ext = args.caption_file_ext
+    overwrite = args.overwrite
+
+    # Get the caption text from either the caption_text or caption_file argument
+    if args.caption_text:
+        caption_text = args.caption_text
+    elif args.caption_file:
+        caption_text = args.caption_file.read()
+
+    # Create a Path object for the image folder
+    folder = Path(image_folder)
+
+    # Check if the image folder exists and is a directory
+    if not folder.is_dir():
+        raise ValueError(f"{image_folder} is not a valid directory.")
+        
+    # Create the caption files
+    create_caption_files(image_folder, file_pattern, caption_text, caption_file_ext, overwrite)
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/tools/cleanup_captions.py b/tools/cleanup_captions.py
new file mode 100644
index 0000000000000000000000000000000000000000..628c0f47c9bf192fd24e1d214474d6cddfb0691d
--- /dev/null
+++ b/tools/cleanup_captions.py
@@ -0,0 +1,27 @@
+import os
+import argparse
+
+parser = argparse.ArgumentParser(description="Remove specified keywords from all text files in a directory.")
+parser.add_argument("folder_path", type=str, help="path to directory containing text files")
+parser.add_argument("-e", "--extension", type=str, default=".txt", help="file extension of text files to be processed (default: .txt)")
+args = parser.parse_args()
+
+folder_path = args.folder_path
+extension = args.extension
+keywords = ["1girl", "solo", "blue eyes", "brown eyes", "blonde hair", "black hair", "realistic", "red lips", "lips", "artist name", "makeup", "realistic","brown hair", "dark skin", 
+            "dark-skinned female", "medium breasts", "breasts", "1boy"]
+
+for file_name in os.listdir(folder_path):
+    if file_name.endswith(extension):
+        file_path = os.path.join(folder_path, file_name)
+        with open(file_path, "r") as f:
+            text = f.read()
+        # extract tags from text and split into a list using comma as the delimiter
+        tags = [tag.strip() for tag in text.split(",")]
+        # remove the specified keywords from the tags list
+        tags = [tag for tag in tags if tag not in keywords]
+        # remove empty or whitespace-only tags
+        tags = [tag for tag in tags if tag.strip() != ""]
+        # join the tags back into a comma-separated string and write back to the file
+        with open(file_path, "w") as f:
+            f.write(", ".join(tags))
\ No newline at end of file
diff --git a/tools/convert_diffusers20_original_sd.md b/tools/convert_diffusers20_original_sd.md
new file mode 100644
index 0000000000000000000000000000000000000000..4763e5fd563e603a31ac5b3b9e085d865be74d32
--- /dev/null
+++ b/tools/convert_diffusers20_original_sd.md
@@ -0,0 +1,46 @@
+# How to use
+
+##Diffusers to Stable Diffusion .ckpt conversion
+
+Specify the folder of the source model and the destination .ckpt file as follows (actually written on one line). The v1/v2 version is automatically determined.
+
+```
+python convert_diffusers20_original_sd.py ..\models\diffusers_model 
+    ..\models\sd.ckpt
+```
+
+Note that v2 Diffusers' Text Encoder has only 22 layers, and if you convert it to Stable Diffusion as it is, the weights will be insufficient, so the weights of the 22 layers will be copied as the 23rd layer. The weight of the 23rd layer is not used during image generation, so it has no effect. Similarly, text_projection logit_scale also adds dummy weights (it doesn't seem to be used for image generation).
+
+## Stable Diffusion .ckpt to Diffusers
+
+Enter the following:
+
+```
+python convert_diffusers20_original_sd.py ..\models\sd.ckpt 
+    ..\models\diffusers_model 
+    --v2 --reference_model stabilityai/stable-diffusion-2
+```
+
+Specify the .ckpt file and the destination folder as arguments.
+Model judgment is not possible, so please use the `--v1` option or the `--v2` option depending on the model.
+
+Also, since `.ckpt` does not contain scheduler and tokenizer information, you need to copy them from some existing Diffusers model. Please specify with `--reference_model`. You can specify the HuggingFace id or a local model directory.
+
+If you don't have a local model, you can specify "stabilityai/stable-diffusion-2" or "stabilityai/stable-diffusion-2-base" for v2.
+For v1.4/1.5, "CompVis/stable-diffusion-v1-4" is fine (v1.4 and v1.5 seem to be the same).
+
+## What can you do?
+
+`--fp16 / --bf16 / --float`
+
+You can specify the data format when saving checkpoint. --fp16 only, also valid when loading Diffusers models.
+
+`--epoch / --global_step`
+
+When saving checkpoint, write epoch and global_step with the specified values. If not specified, both will be 0.
+
+## Conclusion
+
+Some people may be troubled by the Diffusers model due to the poor inference environment. I hope it helps a little.
+
+(Note that converting the data format from checkpoint to checkpoint is also possible, although it has not been tested.) ）
\ No newline at end of file
diff --git a/tools/convert_diffusers20_original_sd.py b/tools/convert_diffusers20_original_sd.py
new file mode 100644
index 0000000000000000000000000000000000000000..b9365b519fc22dee3b4e65de89d59b06e8ccd78c
--- /dev/null
+++ b/tools/convert_diffusers20_original_sd.py
@@ -0,0 +1,133 @@
+# convert Diffusers v1.x/v2.0 model to original Stable Diffusion
+
+import argparse
+import os
+import torch
+from diffusers import StableDiffusionPipeline
+
+import library.model_util as model_util
+
+
+def convert(args):
+    # 引数を確認する
+    load_dtype = torch.float16 if args.fp16 else None
+
+    save_dtype = None
+    if args.fp16 or args.save_precision_as == "fp16":
+        save_dtype = torch.float16
+    elif args.bf16 or args.save_precision_as == "bf16":
+        save_dtype = torch.bfloat16
+    elif args.float or args.save_precision_as == "float":
+        save_dtype = torch.float
+
+    is_load_ckpt = os.path.isfile(args.model_to_load)
+    is_save_ckpt = len(os.path.splitext(args.model_to_save)[1]) > 0
+
+    assert not is_load_ckpt or args.v1 != args.v2, f"v1 or v2 is required to load checkpoint / checkpointの読み込みにはv1/v2指定が必要です"
+    # assert (
+    #     is_save_ckpt or args.reference_model is not None
+    # ), f"reference model is required to save as Diffusers / Diffusers形式での保存には参照モデルが必要です"
+
+    # モデルを読み込む
+    msg = "checkpoint" if is_load_ckpt else ("Diffusers" + (" as fp16" if args.fp16 else ""))
+    print(f"loading {msg}: {args.model_to_load}")
+
+    if is_load_ckpt:
+        v2_model = args.v2
+        text_encoder, vae, unet = model_util.load_models_from_stable_diffusion_checkpoint(v2_model, args.model_to_load, unet_use_linear_projection_in_v2=args.unet_use_linear_projection)
+    else:
+        pipe = StableDiffusionPipeline.from_pretrained(
+            args.model_to_load, torch_dtype=load_dtype, tokenizer=None, safety_checker=None
+        )
+        text_encoder = pipe.text_encoder
+        vae = pipe.vae
+        unet = pipe.unet
+
+        if args.v1 == args.v2:
+            # 自動判定する
+            v2_model = unet.config.cross_attention_dim == 1024
+            print("checking model version: model is " + ("v2" if v2_model else "v1"))
+        else:
+            v2_model = not args.v1
+
+    # 変換して保存する
+    msg = ("checkpoint" + ("" if save_dtype is None else f" in {save_dtype}")) if is_save_ckpt else "Diffusers"
+    print(f"converting and saving as {msg}: {args.model_to_save}")
+
+    if is_save_ckpt:
+        original_model = args.model_to_load if is_load_ckpt else None
+        key_count = model_util.save_stable_diffusion_checkpoint(
+            v2_model, args.model_to_save, text_encoder, unet, original_model, args.epoch, args.global_step, save_dtype, vae
+        )
+        print(f"model saved. total converted state_dict keys: {key_count}")
+    else:
+        print(f"copy scheduler/tokenizer config from: {args.reference_model if args.reference_model is not None else 'default model'}")
+        model_util.save_diffusers_checkpoint(
+            v2_model, args.model_to_save, text_encoder, unet, args.reference_model, vae, args.use_safetensors
+        )
+        print(f"model saved.")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--v1", action="store_true", help="load v1.x model (v1 or v2 is required to load checkpoint) / 1.xのモデルを読み込む"
+    )
+    parser.add_argument(
+        "--v2", action="store_true", help="load v2.0 model (v1 or v2 is required to load checkpoint) / 2.0のモデルを読み込む"
+    )
+    parser.add_argument(
+        "--unet_use_linear_projection", action="store_true", help="When saving v2 model as Diffusers, set U-Net config to `use_linear_projection=true` (to match stabilityai's model) / Diffusers形式でv2モデルを保存するときにU-Netの設定を`use_linear_projection=true`にする（stabilityaiのモデルと合わせる）"
+    )
+    parser.add_argument(
+        "--fp16",
+        action="store_true",
+        help="load as fp16 (Diffusers only) and save as fp16 (checkpoint only) / fp16形式で読み込み（Diffusers形式のみ対応）、保存する（checkpointのみ対応）",
+    )
+    parser.add_argument("--bf16", action="store_true", help="save as bf16 (checkpoint only) / bf16形式で保存する（checkpointのみ対応）")
+    parser.add_argument(
+        "--float", action="store_true", help="save as float (checkpoint only) / float(float32)形式で保存する（checkpointのみ対応）"
+    )
+    parser.add_argument(
+        "--save_precision_as",
+        type=str,
+        default="no",
+        choices=["fp16", "bf16", "float"],
+        help="save precision, do not specify with --fp16/--bf16/--float / 保存する精度、--fp16/--bf16/--floatと併用しないでください",
+    )
+    parser.add_argument("--epoch", type=int, default=0, help="epoch to write to checkpoint / checkpointに記録するepoch数の値")
+    parser.add_argument(
+        "--global_step", type=int, default=0, help="global_step to write to checkpoint / checkpointに記録するglobal_stepの値"
+    )
+    parser.add_argument(
+        "--reference_model",
+        type=str,
+        default=None,
+        help="scheduler/tokenizerのコピー元Diffusersモデル、Diffusers形式で保存するときに使用される、省略時は`runwayml/stable-diffusion-v1-5` または `stabilityai/stable-diffusion-2-1` / reference Diffusers model to copy scheduler/tokenizer config from, used when saving as Diffusers format, default is `runwayml/stable-diffusion-v1-5` or `stabilityai/stable-diffusion-2-1`",
+    )
+    parser.add_argument(
+        "--use_safetensors",
+        action="store_true",
+        help="use safetensors format to save Diffusers model (checkpoint depends on the file extension) / Duffusersモデルをsafetensors形式で保存する（checkpointは拡張子で自動判定）",
+    )
+
+    parser.add_argument(
+        "model_to_load",
+        type=str,
+        default=None,
+        help="model to load: checkpoint file or Diffusers model's directory / 読み込むモデル、checkpointかDiffusers形式モデルのディレクトリ",
+    )
+    parser.add_argument(
+        "model_to_save",
+        type=str,
+        default=None,
+        help="model to save: checkpoint (with extension) or Diffusers model's directory (without extension) / 変換後のモデル、拡張子がある場合はcheckpoint、ない場合はDiffusesモデルとして保存",
+    )
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    convert(args)
diff --git a/tools/convert_images_to_hq_jpg.py b/tools/convert_images_to_hq_jpg.py
new file mode 100644
index 0000000000000000000000000000000000000000..efc40477892194a49ae58f116c6a3db9c46a9cd2
--- /dev/null
+++ b/tools/convert_images_to_hq_jpg.py
@@ -0,0 +1,57 @@
+import argparse
+import glob
+import os
+from pathlib import Path
+from PIL import Image
+
+
+def main():
+    # Define the command-line arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory", type=str,
+                        help="the directory containing the images to be converted")
+    parser.add_argument("--in_ext", type=str, default="webp",
+                        help="the input file extension")
+    parser.add_argument("--quality", type=int, default=95,
+                        help="the JPEG quality (0-100)")
+    parser.add_argument("--delete_originals", action="store_true",
+                        help="whether to delete the original files after conversion")
+
+    # Parse the command-line arguments
+    args = parser.parse_args()
+    directory = args.directory
+    in_ext = args.in_ext
+    out_ext = "jpg"
+    quality = args.quality
+    delete_originals = args.delete_originals
+
+    # Create the file pattern string using the input file extension
+    file_pattern = f"*.{in_ext}"
+
+    # Get the list of files in the directory that match the file pattern
+    files = glob.glob(os.path.join(directory, file_pattern))
+
+    # Iterate over the list of files
+    for file in files:
+        # Open the image file
+        img = Image.open(file)
+
+        # Create a new file path with the output file extension
+        new_path = Path(file).with_suffix(f".{out_ext}")
+
+        # Check if the output file already exists
+        if new_path.exists():
+            # Skip the conversion if the output file already exists
+            print(f"Skipping {file} because {new_path} already exists")
+            continue
+
+        # Save the image to the new file as high-quality JPEG
+        img.save(new_path, quality=quality, optimize=True)
+
+        # Optionally, delete the original file
+        if delete_originals:
+            os.remove(file)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/convert_images_to_webp.py b/tools/convert_images_to_webp.py
new file mode 100644
index 0000000000000000000000000000000000000000..4833459e107d016db39092f63a9fb14a3b8935bf
--- /dev/null
+++ b/tools/convert_images_to_webp.py
@@ -0,0 +1,57 @@
+import argparse
+import glob
+import os
+from pathlib import Path
+from PIL import Image
+
+
+def main():
+    # Define the command-line arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory", type=str,
+                        help="the directory containing the images to be converted")
+    parser.add_argument("--in_ext", type=str, default="webp",
+                        help="the input file extension")
+    parser.add_argument("--delete_originals", action="store_true",
+                        help="whether to delete the original files after conversion")
+
+    # Parse the command-line arguments
+    args = parser.parse_args()
+    directory = args.directory
+    in_ext = args.in_ext
+    delete_originals = args.delete_originals
+
+    # Set the output file extension to .webp
+    out_ext = "webp"
+
+    # Create the file pattern string using the input file extension
+    file_pattern = f"*.{in_ext}"
+
+    # Get the list of files in the directory that match the file pattern
+    files = glob.glob(os.path.join(directory, file_pattern))
+
+    # Iterate over the list of files
+    for file in files:
+        # Open the image file
+        img = Image.open(file)
+
+        # Create a new file path with the output file extension
+        new_path = Path(file).with_suffix(f".{out_ext}")
+        print(new_path)
+
+        # Check if the output file already exists
+        if new_path.exists():
+            # Skip the conversion if the output file already exists
+            print(f"Skipping {file} because {new_path} already exists")
+            continue
+
+        # Save the image to the new file as lossless
+        img.save(new_path, lossless=True)
+
+        # Optionally, delete the original file
+        if delete_originals:
+            os.remove(file)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/crop_images_to_n_buckets.py b/tools/crop_images_to_n_buckets.py
new file mode 100644
index 0000000000000000000000000000000000000000..e2bdbd085c02090c391a8429bf7cd63087b46e06
--- /dev/null
+++ b/tools/crop_images_to_n_buckets.py
@@ -0,0 +1,209 @@
+# This code sorts a collection of images in a given directory by their aspect ratio, groups 
+# them into batches of a given size, crops each image in a batch to the average aspect ratio 
+# of that batch, and saves the cropped images in a specified directory. The user provides 
+# the paths to the input directory and the output directory, as well as the desired batch 
+# size. The program drops any images that do not fit exactly into the batches.
+
+import os
+import cv2
+import argparse
+import shutil
+
+def aspect_ratio(img_path):
+    """Return aspect ratio of an image"""
+    image = cv2.imread(img_path)
+    height, width = image.shape[:2]
+    aspect_ratio = float(width) / float(height)
+    return aspect_ratio
+
+def sort_images_by_aspect_ratio(path):
+    """Sort all images in a folder by aspect ratio"""
+    images = []
+    for filename in os.listdir(path):
+        if filename.endswith(".jpg") or filename.endswith(".jpeg") or filename.endswith(".png") or filename.endswith(".webp"):
+            print(filename)
+            img_path = os.path.join(path, filename)
+            images.append((img_path, aspect_ratio(img_path)))
+    # sort the list of tuples based on the aspect ratio
+    sorted_images = sorted(images, key=lambda x: x[1])
+    return sorted_images
+
+def create_groups(sorted_images, n_groups):
+    """Create n groups from sorted list of images"""
+    n = len(sorted_images)
+    size = n // n_groups
+    groups = [sorted_images[i * size : (i + 1) * size] for i in range(n_groups - 1)]
+    groups.append(sorted_images[(n_groups - 1) * size:])
+    return groups
+
+def average_aspect_ratio(group):
+    """Calculate average aspect ratio for a group"""
+    aspect_ratios = [aspect_ratio for _, aspect_ratio in group]
+    avg_aspect_ratio = sum(aspect_ratios) / len(aspect_ratios)
+    print(f"Average aspect ratio for group: {avg_aspect_ratio}")
+    return avg_aspect_ratio
+
+def center_crop_image(image, target_aspect_ratio):
+    """Crop the input image to the target aspect ratio.
+
+    The function calculates the crop region for the input image based on its current aspect ratio and the target aspect ratio.
+
+    Args:
+        image: A numpy array representing the input image.
+        target_aspect_ratio: A float representing the target aspect ratio.
+
+    Returns:
+        A numpy array representing the cropped image.
+
+    """
+    height, width = image.shape[:2]
+    current_aspect_ratio = float(width) / float(height)
+
+    if current_aspect_ratio == target_aspect_ratio:
+        return image
+
+    if current_aspect_ratio > target_aspect_ratio:
+        new_width = int(target_aspect_ratio * height)
+        x_start = (width - new_width) // 2
+        cropped_image = image[:, x_start:x_start+new_width]
+    else:
+        new_height = int(width / target_aspect_ratio)
+        y_start = (height - new_height) // 2
+        cropped_image = image[y_start:y_start+new_height, :]
+
+    return cropped_image
+
+def copy_related_files(img_path, save_path):
+    """
+    Copy all files in the same directory as the input image that have the same base name as the input image to the
+    output directory with the corresponding new filename.
+    :param img_path: Path to the input image.
+    :param save_path: Path to the output image.
+    """
+    # Get the base filename and directory
+    img_dir, img_basename = os.path.split(img_path)
+    img_base, img_ext = os.path.splitext(img_basename)
+
+    save_dir, save_basename = os.path.split(save_path)
+    save_base, save_ext = os.path.splitext(save_basename)
+
+    # Create the output directory if it does not exist
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+
+    # Loop over all files in the same directory as the input image
+    try:
+        for filename in os.listdir(img_dir):
+            # Skip files with the same name as the input image
+            if filename == img_basename:
+                continue
+
+            # Check if the file has the same base name as the input image
+            file_base, file_ext = os.path.splitext(filename)
+            if file_base == img_base:
+                # Build the new filename and copy the file
+                new_filename = os.path.join(save_dir, f"{save_base}{file_ext}")
+                shutil.copy2(os.path.join(img_dir, filename), new_filename)
+    except OSError as e:
+        print(f"Error: {e}")  # Handle errors from os.listdir()
+
+def save_resized_cropped_images(group, folder_name, group_number, avg_aspect_ratio, use_original_name=False):
+    """Crop and resize all images in the input group to the smallest resolution, and save them to a folder.
+
+    Args:
+        group: A list of tuples, where each tuple contains the path to an image and its aspect ratio.
+        folder_name: A string representing the name of the folder to save the images to.
+        group_number: An integer representing the group number.
+        avg_aspect_ratio: A float representing the average aspect ratio of the images in the group.
+        use_original_name: A boolean indicating whether to save the images with their original file names.
+
+    """
+    if not os.path.exists(folder_name):
+        os.makedirs(folder_name)
+
+    # get the smallest size of the images
+    smallest_res = float("inf")
+    for img_path, _ in group:
+        image = cv2.imread(img_path)
+        cropped_image = center_crop_image(image, avg_aspect_ratio)
+        height, width = cropped_image.shape[:2]
+        image_res = height * width
+        if image_res < smallest_res:
+            smallest_res = image_res
+            small_height, small_width = height, width
+
+    # resize all images to the smallest resolution of the images in the group
+    for i, (img_path, aspect_ratio) in enumerate(group):
+        image = cv2.imread(img_path)
+        cropped_image = center_crop_image(image, avg_aspect_ratio)
+        # resized_image = cv2.resize(cropped_image, (small_width, small_height))
+        if use_original_name:
+            save_name = os.path.basename(img_path)
+        else:
+            save_name = f"group_{group_number}_{i}.jpg"
+        save_path = os.path.join(folder_name, save_name)
+        cv2.imwrite(save_path, cropped_image)
+        
+        # Copy matching files named the same as img_path to
+        copy_related_files(img_path, save_path)
+        
+        print(f"Saved {save_name} to {folder_name}")
+        
+
+def main():
+    parser = argparse.ArgumentParser(description='Sort images and crop them based on aspect ratio')
+    parser.add_argument('input_dir', type=str, help='Path to the directory containing images')
+    parser.add_argument('output_dir', type=str, help='Path to the directory to save the cropped images')
+    parser.add_argument('batch_size', type=int, help='Size of the batches to create')
+    parser.add_argument('--use_original_name', action='store_true', help='Whether to use original file names for the saved images')
+
+    args = parser.parse_args()
+
+    print(f"Sorting images by aspect ratio in {args.input_dir}...")
+    if not os.path.exists(args.input_dir):
+        print(f"Error: Input directory does not exist: {args.input_dir}")
+        return
+
+    if not os.path.exists(args.output_dir):
+        try:
+            os.makedirs(args.output_dir)
+        except OSError:
+            print(f"Error: Failed to create output directory: {args.output_dir}")
+            return
+
+    sorted_images = sort_images_by_aspect_ratio(args.input_dir)
+    total_images = len(sorted_images)
+    print(f'Total images: {total_images}')
+
+    if args.batch_size <= 0:
+        print("Error: Batch size must be greater than 0")
+        return
+    
+    group_size = total_images // args.batch_size
+    
+    print(f'Train batch size: {args.batch_size}, image group size: {group_size}')
+    remainder = total_images % args.batch_size
+
+    if remainder != 0:
+        print(f'Dropping {remainder} images that do not fit in groups...')
+        sorted_images = sorted_images[:-remainder]
+        total_images = len(sorted_images)
+        group_size = total_images // args.batch_size
+
+    print('Creating groups...')
+    groups = create_groups(sorted_images, group_size)
+    print(f"Created {len(groups)} groups")
+
+    print('Saving cropped and resize images...')
+    for i, group in enumerate(groups):
+        avg_aspect_ratio = average_aspect_ratio(group)
+        print(f"Processing group {i+1} with {len(group)} images...")
+        try:
+            save_resized_cropped_images(group, args.output_dir, i+1, avg_aspect_ratio, args.use_original_name)
+        except Exception as e:
+            print(f"Error: Failed to save images in group {i+1}: {e}")
+
+    print('Done')
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/tools/detect_face_rotate.py b/tools/detect_face_rotate.py
new file mode 100644
index 0000000000000000000000000000000000000000..68dec6cae932e827e79c49992238be7fd2edf21c
--- /dev/null
+++ b/tools/detect_face_rotate.py
@@ -0,0 +1,246 @@
+# このスクリプトのライセンスは、train_dreambooth.pyと同じくApache License 2.0とします
+# (c) 2022 Kohya S. @kohya_ss
+
+# 横長の画像から顔検出して正立するように回転し、そこを中心に正方形に切り出す
+
+# v2: extract max face if multiple faces are found
+# v3: add crop_ratio option
+# v4: add multiple faces extraction and min/max size
+
+import argparse
+import math
+import cv2
+import glob
+import os
+from anime_face_detector import create_detector
+from tqdm import tqdm
+import numpy as np
+
+KP_REYE = 11
+KP_LEYE = 19
+
+SCORE_THRES = 0.90
+
+
+def detect_faces(detector, image, min_size):
+  preds = detector(image)                     # bgr
+  # print(len(preds))
+
+  faces = []
+  for pred in preds:
+    bb = pred['bbox']
+    score = bb[-1]
+    if score < SCORE_THRES:
+      continue
+
+    left, top, right, bottom = bb[:4]
+    cx = int((left + right) / 2)
+    cy = int((top + bottom) / 2)
+    fw = int(right - left)
+    fh = int(bottom - top)
+
+    lex, ley = pred['keypoints'][KP_LEYE, 0:2]
+    rex, rey = pred['keypoints'][KP_REYE, 0:2]
+    angle = math.atan2(ley - rey, lex - rex)
+    angle = angle / math.pi * 180
+
+    faces.append((cx, cy, fw, fh, angle))
+
+  faces.sort(key=lambda x: max(x[2], x[3]), reverse=True)         # 大きい順
+  return faces
+
+
+def rotate_image(image, angle, cx, cy):
+  h, w = image.shape[0:2]
+  rot_mat = cv2.getRotationMatrix2D((cx, cy), angle, 1.0)
+
+  # # 回転する分、すこし画像サイズを大きくする→とりあえず無効化
+  # nh = max(h, int(w * math.sin(angle)))
+  # nw = max(w, int(h * math.sin(angle)))
+  # if nh > h or nw > w:
+  #   pad_y = nh - h
+  #   pad_t = pad_y // 2
+  #   pad_x = nw - w
+  #   pad_l = pad_x // 2
+  #   m = np.array([[0, 0, pad_l],
+  #                 [0, 0, pad_t]])
+  #   rot_mat = rot_mat + m
+  #   h, w = nh, nw
+  #   cx += pad_l
+  #   cy += pad_t
+
+  result = cv2.warpAffine(image, rot_mat, (w, h), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)
+  return result, cx, cy
+
+
+def process(args):
+  assert (not args.resize_fit) or args.resize_face_size is None, f"resize_fit and resize_face_size can't be specified both / resize_fitとresize_face_sizeはどちらか片方しか指定できません"
+  assert args.crop_ratio is None or args.resize_face_size is None, f"crop_ratio指定時はresize_face_sizeは指定できません"
+
+  # アニメ顔検出モデルを読み込む
+  print("loading face detector.")
+  detector = create_detector('yolov3')
+
+  # cropの引数を解析する
+  if args.crop_size is None:
+    crop_width = crop_height = None
+  else:
+    tokens = args.crop_size.split(',')
+    assert len(tokens) == 2, f"crop_size must be 'width,height' / crop_sizeは'幅,高さ'で指定してください"
+    crop_width, crop_height = [int(t) for t in tokens]
+
+  if args.crop_ratio is None:
+    crop_h_ratio = crop_v_ratio = None
+  else:
+    tokens = args.crop_ratio.split(',')
+    assert len(tokens) == 2, f"crop_ratio must be 'horizontal,vertical' / crop_ratioは'幅,高さ'の倍率で指定してください"
+    crop_h_ratio, crop_v_ratio = [float(t) for t in tokens]
+
+  # 画像を処理する
+  print("processing.")
+  output_extension = ".png"
+
+  os.makedirs(args.dst_dir, exist_ok=True)
+  paths = glob.glob(os.path.join(args.src_dir, "*.png")) + glob.glob(os.path.join(args.src_dir, "*.jpg")) + \
+      glob.glob(os.path.join(args.src_dir, "*.webp"))
+  for path in tqdm(paths):
+    basename = os.path.splitext(os.path.basename(path))[0]
+
+    # image = cv2.imread(path)        # 日本語ファイル名でエラーになる
+    image = cv2.imdecode(np.fromfile(path, np.uint8), cv2.IMREAD_UNCHANGED)
+    if len(image.shape) == 2:
+      image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
+    if image.shape[2] == 4:
+      print(f"image has alpha. ignore / 画像の透明度が設定されているため無視します: {path}")
+      image = image[:, :, :3].copy()                    # copyをしないと内部的に透明度情報が付いたままになるらしい
+
+    h, w = image.shape[:2]
+
+    faces = detect_faces(detector, image, args.multiple_faces)
+    for i, face in enumerate(faces):
+      cx, cy, fw, fh, angle = face
+      face_size = max(fw, fh)
+      if args.min_size is not None and face_size < args.min_size:
+        continue
+      if args.max_size is not None and face_size >= args.max_size:
+        continue
+      face_suffix = f"_{i+1:02d}" if args.multiple_faces else ""
+
+      # オプション指定があれば回転する
+      face_img = image
+      if args.rotate:
+        face_img, cx, cy = rotate_image(face_img, angle, cx, cy)
+
+      # オプション指定があれば顔を中心に切り出す
+      if crop_width is not None or crop_h_ratio is not None:
+        cur_crop_width, cur_crop_height = crop_width, crop_height
+        if crop_h_ratio is not None:
+          cur_crop_width = int(face_size * crop_h_ratio + .5)
+          cur_crop_height = int(face_size * crop_v_ratio + .5)
+
+        # リサイズを必要なら行う
+        scale = 1.0
+        if args.resize_face_size is not None:
+          # 顔サイズを基準にリサイズする
+          scale = args.resize_face_size / face_size
+          if scale < cur_crop_width / w:
+            print(
+                f"image width too small in face size based resizing / 顔を基準にリサイズすると画像の幅がcrop sizeより小さい（顔が相対的に大きすぎる）ので顔サイズが変わります: {path}")
+            scale = cur_crop_width / w
+          if scale < cur_crop_height / h:
+            print(
+                f"image height too small in face size based resizing / 顔を基準にリサイズすると画像の高さがcrop sizeより小さい（顔が相対的に大きすぎる）ので顔サイズが変わります: {path}")
+            scale = cur_crop_height / h
+        elif crop_h_ratio is not None:
+          # 倍率指定の時にはリサイズしない
+          pass
+        else:
+          # 切り出しサイズ指定あり
+          if w < cur_crop_width:
+            print(f"image width too small/ 画像の幅がcrop sizeより小さいので画質が劣化します: {path}")
+            scale = cur_crop_width / w
+          if h < cur_crop_height:
+            print(f"image height too small/ 画像の高さがcrop sizeより小さいので画質が劣化します: {path}")
+            scale = cur_crop_height / h
+          if args.resize_fit:
+            scale = max(cur_crop_width / w, cur_crop_height / h)
+
+        if scale != 1.0:
+          w = int(w * scale + .5)
+          h = int(h * scale + .5)
+          face_img = cv2.resize(face_img, (w, h), interpolation=cv2.INTER_AREA if scale < 1.0 else cv2.INTER_LANCZOS4)
+          cx = int(cx * scale + .5)
+          cy = int(cy * scale + .5)
+          fw = int(fw * scale + .5)
+          fh = int(fh * scale + .5)
+
+        cur_crop_width = min(cur_crop_width, face_img.shape[1])
+        cur_crop_height = min(cur_crop_height, face_img.shape[0])
+
+        x = cx - cur_crop_width // 2
+        cx = cur_crop_width // 2
+        if x < 0:
+          cx = cx + x
+          x = 0
+        elif x + cur_crop_width > w:
+          cx = cx + (x + cur_crop_width - w)
+          x = w - cur_crop_width
+        face_img = face_img[:, x:x+cur_crop_width]
+
+        y = cy - cur_crop_height // 2
+        cy = cur_crop_height // 2
+        if y < 0:
+          cy = cy + y
+          y = 0
+        elif y + cur_crop_height > h:
+          cy = cy + (y + cur_crop_height - h)
+          y = h - cur_crop_height
+        face_img = face_img[y:y + cur_crop_height]
+
+      # # debug
+      # print(path, cx, cy, angle)
+      # crp = cv2.resize(image, (image.shape[1]//8, image.shape[0]//8))
+      # cv2.imshow("image", crp)
+      # if cv2.waitKey() == 27:
+      #   break
+      # cv2.destroyAllWindows()
+
+      # debug
+      if args.debug:
+        cv2.rectangle(face_img, (cx-fw//2, cy-fh//2), (cx+fw//2, cy+fh//2), (255, 0, 255), fw//20)
+
+      _, buf = cv2.imencode(output_extension, face_img)
+      with open(os.path.join(args.dst_dir, f"{basename}{face_suffix}_{cx:04d}_{cy:04d}_{fw:04d}_{fh:04d}{output_extension}"), "wb") as f:
+        buf.tofile(f)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--src_dir", type=str, help="directory to load images / 画像を読み込むディレクトリ")
+  parser.add_argument("--dst_dir", type=str, help="directory to save images / 画像を保存するディレクトリ")
+  parser.add_argument("--rotate", action="store_true", help="rotate images to align faces / 顔が正立するように画像を回転する")
+  parser.add_argument("--resize_fit", action="store_true",
+                      help="resize to fit smaller side after cropping / 切り出し後の画像の短辺がcrop_sizeにあうようにリサイズする")
+  parser.add_argument("--resize_face_size", type=int, default=None,
+                      help="resize image before cropping by face size / 切り出し前に顔がこのサイズになるようにリサイズする")
+  parser.add_argument("--crop_size", type=str, default=None,
+                      help="crop images with 'width,height' pixels, face centered / 顔を中心として'幅,高さ'のサイズで切り出す")
+  parser.add_argument("--crop_ratio", type=str, default=None,
+                      help="crop images with 'horizontal,vertical' ratio to face, face centered / 顔を中心として顔サイズの'幅倍率,高さ倍率'のサイズで切り出す")
+  parser.add_argument("--min_size", type=int, default=None,
+                      help="minimum face size to output (included) / 処理対象とする顔の最小サイズ（この値以上）")
+  parser.add_argument("--max_size", type=int, default=None,
+                      help="maximum face size to output (excluded) / 処理対象とする顔の最大サイズ（この値未満）")
+  parser.add_argument("--multiple_faces", action="store_true",
+                      help="output each faces / 複数の顔が見つかった場合、それぞれを切り出す")
+  parser.add_argument("--debug", action="store_true", help="render rect for face / 処理後画像の顔位置に矩形を描画します")
+
+  return parser
+
+
+if __name__ == '__main__':
+  parser = setup_parser()
+
+  args = parser.parse_args()
+
+  process(args)
diff --git a/tools/extract_locon.py b/tools/extract_locon.py
new file mode 100644
index 0000000000000000000000000000000000000000..ca72166518a74df0ecf3882e89ecf6b429ba5c0f
--- /dev/null
+++ b/tools/extract_locon.py
@@ -0,0 +1,106 @@
+#
+# From: https://raw.githubusercontent.com/KohakuBlueleaf/LoCon/main/extract_locon.py
+#
+
+import argparse
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "base_model", help="The model which use it to train the dreambooth model",
+        default='', type=str
+    )
+    parser.add_argument(
+        "db_model", help="the dreambooth model you want to extract the locon",
+        default='', type=str
+    )
+    parser.add_argument(
+        "output_name", help="the output model",
+        default='./out.pt', type=str
+    )
+    parser.add_argument(
+        "--is_v2", help="Your base/db model is sd v2 or not",
+        default=False, action="store_true"
+    )
+    parser.add_argument(
+        "--device", help="Which device you want to use to extract the locon",
+        default='cpu', type=str
+    )
+    parser.add_argument(
+        "--mode", 
+        help=(
+            'extraction mode, can be "fixed", "threshold", "ratio", "percentile". '
+            'If not "fixed", network_dim and conv_dim will be ignored'
+        ),
+        default='fixed', type=str
+    )
+    parser.add_argument(
+        "--linear_dim", help="network dim for linear layer in fixed mode",
+        default=1, type=int
+    )
+    parser.add_argument(
+        "--conv_dim", help="network dim for conv layer in fixed mode",
+        default=1, type=int
+    )
+    parser.add_argument(
+        "--linear_threshold", help="singular value threshold for linear layer in threshold mode",
+        default=0., type=float
+    )
+    parser.add_argument(
+        "--conv_threshold", help="singular value threshold for conv layer in threshold mode",
+        default=0., type=float
+    )
+    parser.add_argument(
+        "--linear_ratio", help="singular ratio for linear layer in ratio mode",
+        default=0., type=float
+    )
+    parser.add_argument(
+        "--conv_ratio", help="singular ratio for conv layer in ratio mode",
+        default=0., type=float
+    )
+    parser.add_argument(
+        "--linear_percentile", help="singular value percentile for linear layer percentile mode",
+        default=1., type=float
+    )
+    parser.add_argument(
+        "--conv_percentile", help="singular value percentile for conv layer percentile mode",
+        default=1., type=float
+    )
+    return parser.parse_args()
+ARGS = get_args()
+
+from locon.utils import extract_diff
+from locon.kohya_model_utils import load_models_from_stable_diffusion_checkpoint
+
+import torch
+
+
+def main():
+    args = ARGS
+    base = load_models_from_stable_diffusion_checkpoint(args.is_v2, args.base_model)
+    db = load_models_from_stable_diffusion_checkpoint(args.is_v2, args.db_model)
+    
+    linear_mode_param = {
+        'fixed': args.linear_dim,
+        'threshold': args.linear_threshold,
+        'ratio': args.linear_ratio,
+        'percentile': args.linear_percentile,
+    }[args.mode]
+    conv_mode_param = {
+        'fixed': args.conv_dim,
+        'threshold': args.conv_threshold,
+        'ratio': args.conv_ratio,
+        'percentile': args.conv_percentile,
+    }[args.mode]
+    
+    state_dict = extract_diff(
+        base, db,
+        args.mode,
+        linear_mode_param, conv_mode_param,
+        args.device
+    )
+    torch.save(state_dict, args.output_name)
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/tools/gradio_theme_builder.py b/tools/gradio_theme_builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..c10965535c03d58f63e97e64f5dfb88dfe569ae3
--- /dev/null
+++ b/tools/gradio_theme_builder.py
@@ -0,0 +1,2 @@
+import gradio as gr
+gr.themes.builder()
diff --git a/tools/group_images.py b/tools/group_images.py
new file mode 100644
index 0000000000000000000000000000000000000000..4bc420f47248ee061abed967f0261a36c1e3ef1d
--- /dev/null
+++ b/tools/group_images.py
@@ -0,0 +1,183 @@
+import argparse
+import shutil
+from PIL import Image, ImageOps
+import os
+import numpy as np
+
+from library.custom_logging import setup_logging
+
+# Set up logging
+log = setup_logging()
+
+class ImageProcessor:
+
+    def __init__(self, input_folder, output_folder, group_size, include_subfolders, do_not_copy_other_files, pad, caption, caption_ext):
+        self.input_folder = input_folder
+        self.output_folder = output_folder
+        self.group_size = group_size
+        self.include_subfolders = include_subfolders
+        self.do_not_copy_other_files = do_not_copy_other_files
+        self.pad = pad
+        self.caption = caption
+        self.caption_ext = caption_ext
+        self.image_extensions = ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.tiff')
+
+    def get_image_paths(self):
+        images = []
+        if self.include_subfolders:
+            for dirpath, dirnames, filenames in os.walk(self.input_folder):
+                for filename in filenames:
+                    if filename.endswith(self.image_extensions):
+                        images.append(os.path.join(dirpath, filename))
+        else:
+            images = [os.path.join(self.input_folder, f) for f in os.listdir(self.input_folder) if f.endswith(self.image_extensions)]
+        return images
+
+    def group_images(self, images):
+        sorted_images = sorted(images, key=lambda path: Image.open(path).size[0] / Image.open(path).size[1])
+        groups = [sorted_images[i:i+self.group_size] for i in range(0, len(sorted_images), self.group_size)]
+        return groups
+
+    def process_group(self, group, group_index):
+        if len(group) > 0:
+            aspect_ratios = self.get_aspect_ratios(group)
+            avg_aspect_ratio = np.mean(aspect_ratios)
+            if self.pad:
+                padded_images = self.pad_images(group, avg_aspect_ratio)
+                self.resize_and_save_images(padded_images, group_index, group)
+            else:
+                cropped_images = self.crop_images(group, avg_aspect_ratio)
+                self.resize_and_save_images(cropped_images, group_index, group)
+            if not self.do_not_copy_other_files:
+                self.copy_other_files(group, group_index)
+
+    def get_aspect_ratios(self, group):
+        aspect_ratios = []
+        for path in group:
+            with Image.open(path) as img:
+                width, height = img.size
+                aspect_ratios.append(width / height)
+        return aspect_ratios
+
+    def crop_images(self, group, avg_aspect_ratio):
+        cropped_images = []
+        for j, path in enumerate(group):
+            with Image.open(path) as img:
+                log.info(f"  Processing image {j+1}: {path}")
+                img = self.crop_image(img, avg_aspect_ratio)
+                cropped_images.append(img)
+        return cropped_images
+
+    def crop_image(self, img, avg_aspect_ratio):
+        img_aspect_ratio = img.width / img.height
+        if img_aspect_ratio > avg_aspect_ratio:
+            # Too wide, reduce width
+            new_width = avg_aspect_ratio * img.height
+            left = (img.width - new_width) / 2
+            right = left + new_width
+            img = img.crop((left, 0, right, img.height))
+        else:
+            # Too tall, reduce height
+            new_height = img.width / avg_aspect_ratio
+            top = (img.height - new_height) / 2
+            bottom = top + new_height
+            img = img.crop((0, top, img.width, bottom))
+        return img
+
+    def resize_and_save_images(self, cropped_images, group_index, source_paths):
+        max_width = max(img.width for img in cropped_images)
+        max_height = max(img.height for img in cropped_images)
+        for j, img in enumerate(cropped_images):
+            img = img.resize((max_width, max_height))
+            os.makedirs(self.output_folder, exist_ok=True)
+            original_filename = os.path.basename(source_paths[j])
+            filename_without_ext = os.path.splitext(original_filename)[0]
+            final_file_name = f"group-{group_index+1}-{j+1}-{filename_without_ext}"
+            output_path = os.path.join(self.output_folder, f"{final_file_name}.jpg")
+            log.info(f"  Saving processed image to {output_path}")
+            img.convert('RGB').save(output_path, quality=100)
+            
+            if self.caption:
+                self.create_caption_file(source_paths[j], group_index, final_file_name)
+
+    def create_caption_file(self, source_path, group_index, caption_filename):
+        dirpath = os.path.dirname(source_path)
+        caption = os.path.basename(dirpath).split('_')[-1]
+        caption_filename = caption_filename + self.caption_ext
+        caption_path = os.path.join(self.output_folder, caption_filename)
+        with open(caption_path, 'w') as f:
+            f.write(caption)
+
+
+    def copy_other_files(self, group, group_index):
+        for j, path in enumerate(group):
+            dirpath, original_filename = os.path.split(path)
+            original_basename, original_ext = os.path.splitext(original_filename)
+            for filename in os.listdir(dirpath):
+                if filename.endswith('.npz'):  # Skip .npz
+                    continue
+                basename, ext = os.path.splitext(filename)
+                if basename == original_basename and ext != original_ext:
+                    shutil.copy2(os.path.join(dirpath, filename), os.path.join(self.output_folder, f"group-{group_index+1}-{j+1}-{filename}"))
+
+    def process_images(self):
+        images = self.get_image_paths()
+        groups = self.group_images(images)
+        for i, group in enumerate(groups):
+            log.info(f"Processing group {i+1} with {len(group)} images...")
+            self.process_group(group, i)
+            
+    def process_group(self, group, group_index):
+        if len(group) > 0:
+            aspect_ratios = self.get_aspect_ratios(group)
+            avg_aspect_ratio = np.mean(aspect_ratios)
+            if self.pad:
+                padded_images = self.pad_images(group, avg_aspect_ratio)
+                self.resize_and_save_images(padded_images, group_index, group)
+            else:
+                cropped_images = self.crop_images(group, avg_aspect_ratio)
+                self.resize_and_save_images(cropped_images, group_index, group)
+            if not self.do_not_copy_other_files:
+                self.copy_other_files(group, group_index)
+
+    def pad_images(self, group, avg_aspect_ratio):
+        padded_images = []
+        for j, path in enumerate(group):
+            with Image.open(path) as img:
+                log.info(f"  Processing image {j+1}: {path}")
+                img = self.pad_image(img, avg_aspect_ratio)
+                padded_images.append(img)
+        return padded_images
+
+    def pad_image(self, img, avg_aspect_ratio):
+        img_aspect_ratio = img.width / img.height
+        if img_aspect_ratio < avg_aspect_ratio:
+            # Too tall, increase width
+            new_width = avg_aspect_ratio * img.height
+            pad_width = int((new_width - img.width) / 2)
+            img = ImageOps.expand(img, border=(pad_width, 0), fill='black')
+        else:
+            # Too wide, increase height
+            new_height = img.width / avg_aspect_ratio
+            pad_height = int((new_height - img.height) / 2)
+            img = ImageOps.expand(img, border=(0, pad_height), fill='black')
+        return img
+
+def main():
+    parser = argparse.ArgumentParser(description='Process groups of images.')
+    parser.add_argument('input_folder', type=str, help='Input folder containing images')
+    parser.add_argument('output_folder', type=str, help='Output folder to store processed images')
+    parser.add_argument('group_size', type=int, help='Number of images in each group')
+    parser.add_argument('--include_subfolders', action='store_true', help='Include subfolders in search for images')
+    parser.add_argument('--do_not_copy_other_files', '--no_copy', dest='do_not_copy_other_files', action='store_true', help='Do not copy other files with the same name as images')
+    parser.add_argument('--pad', action='store_true', help='Pad images instead of cropping them')
+    parser.add_argument('--caption', action='store_true', help='Create a caption file for each image')
+    parser.add_argument('--caption_ext', type=str, default='.txt', help='Extension for the caption file')
+
+    args = parser.parse_args()
+
+    processor = ImageProcessor(args.input_folder, args.output_folder, args.group_size, args.include_subfolders, args.do_not_copy_other_files, args.pad, args.caption, args.caption_ext)
+    processor.process_images()
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/group_images_recommended_size.py b/tools/group_images_recommended_size.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac0d8a2f1b03126c7ba0481694e0a71708153e1e
--- /dev/null
+++ b/tools/group_images_recommended_size.py
@@ -0,0 +1,128 @@
+import argparse
+from PIL import Image
+import os
+import numpy as np
+import itertools
+
+class ImageProcessor:
+
+    def __init__(self, input_folder, min_group, max_group, include_subfolders, pad):
+        self.input_folder = input_folder
+        self.min_group = min_group
+        self.max_group = max_group
+        self.include_subfolders = include_subfolders
+        self.pad = pad
+        self.image_extensions = ('.png', '.jpg', '.jpeg', '.gif', '.webp')
+        self.losses = []  # List to store loss values for each image
+
+    def get_image_paths(self):
+        images = []
+        if self.include_subfolders:
+            for dirpath, dirnames, filenames in os.walk(self.input_folder):
+                for filename in filenames:
+                    if filename.endswith(self.image_extensions):
+                        images.append(os.path.join(dirpath, filename))
+        else:
+            images = [os.path.join(self.input_folder, f) for f in os.listdir(self.input_folder) if f.endswith(self.image_extensions)]
+        return images
+
+    def group_images(self, images, group_size):
+        sorted_images = sorted(images, key=lambda path: Image.open(path).size[0] / Image.open(path).size[1])
+        groups = [sorted_images[i:i+group_size] for i in range(0, len(sorted_images), group_size)]
+        return groups
+
+    def process_group(self, group):
+        if len(group) > 0:
+            aspect_ratios = self.get_aspect_ratios(group)
+            avg_aspect_ratio = np.mean(aspect_ratios)
+            self.calculate_losses(group, avg_aspect_ratio)
+
+    def get_aspect_ratios(self, group):
+        aspect_ratios = []
+        for path in group:
+            with Image.open(path) as img:
+                width, height = img.size
+                aspect_ratios.append(width / height)
+        return aspect_ratios
+
+    def calculate_losses(self, group, avg_aspect_ratio):
+        for j, path in enumerate(group):
+            with Image.open(path) as img:
+                loss = self.calculate_loss(img, avg_aspect_ratio)
+                self.losses.append((path, loss))  # Add (path, loss) tuple to the list
+
+    def calculate_loss(self, img, avg_aspect_ratio):
+        img_aspect_ratio = img.width / img.height
+        if img_aspect_ratio > avg_aspect_ratio:
+            # Too wide, reduce width
+            new_width = avg_aspect_ratio * img.height
+            loss = abs(img.width - new_width) / img.width  # Calculate loss value
+        else:
+            # Too tall, reduce height
+            new_height = img.width / avg_aspect_ratio
+            loss = abs(img.height - new_height) / img.height  # Calculate loss value
+        return loss
+
+    def monte_carlo_optimization(self, groups):
+        best_groups = groups.copy()
+        best_loss = np.inf
+        best_removed_images = []
+
+        for group in groups:
+            num_images = len(group)
+            all_combinations = []
+            # Generate all possible combinations of images to remove
+            for r in range(1, num_images + 1):
+                combinations = list(itertools.combinations(group, r))
+                all_combinations.extend(combinations)
+
+            for combination in all_combinations:
+                self.losses = []  # Reset losses for each combination
+                remaining_images = list(set(group) - set(combination))
+                self.process_group(remaining_images)
+                avg_loss = np.mean(self.losses)
+
+                if avg_loss < best_loss:
+                    best_loss = avg_loss
+                    best_groups[best_groups.index(group)] = remaining_images
+                    best_removed_images = combination
+
+        return best_groups, best_loss, best_removed_images
+
+    def process_images(self):
+        images = self.get_image_paths()
+        num_images = len(images)
+        results = []
+
+        for group_size in range(self.min_group, self.max_group + 1):
+            groups = self.group_images(images, group_size)
+            optimized_groups, avg_loss, removed_images = self.monte_carlo_optimization(groups)
+            num_remaining = num_images % group_size
+
+            results.append((group_size, avg_loss, num_remaining, optimized_groups, removed_images))
+
+        # Sort results based on average crop loss in ascending order
+        sorted_results = sorted(results, key=lambda x: x[1])
+
+        for group_size, avg_loss, num_remaining, optimized_groups, removed_images in sorted_results:
+            print(f"Group size: {group_size}, Average crop loss: {avg_loss}, Number of images remaining: {num_remaining}")
+            print(f"Optimized Groups: {optimized_groups}")
+            print(f"Removed Images: {removed_images}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Process groups of images.')
+    parser.add_argument('input_folder', type=str, help='Input folder containing images')
+    parser.add_argument('min_group', type=int, help='Minimum group size')
+    parser.add_argument('max_group', type=int, help='Maximum group size')
+    parser.add_argument('--include_subfolders', action='store_true', help='Include subfolders in search for images')
+    parser.add_argument('--pad', action='store_true', help='Pad images instead of cropping them')
+
+    args = parser.parse_args()
+
+    processor = ImageProcessor(args.input_folder, args.min_group, args.max_group, args.include_subfolders, args.pad)
+    processor.process_images()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/latent_upscaler.py b/tools/latent_upscaler.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab1fa3390fa781c24fe796f1ff0987c91dcc3f95
--- /dev/null
+++ b/tools/latent_upscaler.py
@@ -0,0 +1,348 @@
+# 外部から簡単にupscalerを呼ぶためのスクリプト
+# 単体で動くようにモデル定義も含めている
+
+import argparse
+import glob
+import os
+import cv2
+from diffusers import AutoencoderKL
+
+from typing import Dict, List
+import numpy as np
+
+import torch
+from torch import nn
+from tqdm import tqdm
+from PIL import Image
+
+
+class ResidualBlock(nn.Module):
+    def __init__(self, in_channels, out_channels=None, kernel_size=3, stride=1, padding=1):
+        super(ResidualBlock, self).__init__()
+
+        if out_channels is None:
+            out_channels = in_channels
+
+        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias=False)
+        self.bn1 = nn.BatchNorm2d(out_channels)
+        self.relu1 = nn.ReLU(inplace=True)
+
+        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size, stride, padding, bias=False)
+        self.bn2 = nn.BatchNorm2d(out_channels)
+
+        self.relu2 = nn.ReLU(inplace=True)  # このReLUはresidualに足す前にかけるほうがいいかも
+
+        # initialize weights
+        self._initialize_weights()
+
+    def _initialize_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
+                if m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.constant_(m.bias, 0)
+
+    def forward(self, x):
+        residual = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu1(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+
+        out += residual
+
+        out = self.relu2(out)
+
+        return out
+
+
+class Upscaler(nn.Module):
+    def __init__(self):
+        super(Upscaler, self).__init__()
+
+        # define layers
+        # latent has 4 channels
+
+        self.conv1 = nn.Conv2d(4, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+        self.bn1 = nn.BatchNorm2d(128)
+        self.relu1 = nn.ReLU(inplace=True)
+
+        # resblocks
+        # 数の暴力で20個：次元数を増やすよりもブロックを増やしたほうがreceptive fieldが広がるはずだぞ
+        self.resblock1 = ResidualBlock(128)
+        self.resblock2 = ResidualBlock(128)
+        self.resblock3 = ResidualBlock(128)
+        self.resblock4 = ResidualBlock(128)
+        self.resblock5 = ResidualBlock(128)
+        self.resblock6 = ResidualBlock(128)
+        self.resblock7 = ResidualBlock(128)
+        self.resblock8 = ResidualBlock(128)
+        self.resblock9 = ResidualBlock(128)
+        self.resblock10 = ResidualBlock(128)
+        self.resblock11 = ResidualBlock(128)
+        self.resblock12 = ResidualBlock(128)
+        self.resblock13 = ResidualBlock(128)
+        self.resblock14 = ResidualBlock(128)
+        self.resblock15 = ResidualBlock(128)
+        self.resblock16 = ResidualBlock(128)
+        self.resblock17 = ResidualBlock(128)
+        self.resblock18 = ResidualBlock(128)
+        self.resblock19 = ResidualBlock(128)
+        self.resblock20 = ResidualBlock(128)
+
+        # last convs
+        self.conv2 = nn.Conv2d(128, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+        self.bn2 = nn.BatchNorm2d(64)
+        self.relu2 = nn.ReLU(inplace=True)
+
+        self.conv3 = nn.Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+        self.bn3 = nn.BatchNorm2d(64)
+        self.relu3 = nn.ReLU(inplace=True)
+
+        # final conv: output 4 channels
+        self.conv_final = nn.Conv2d(64, 4, kernel_size=(1, 1), stride=(1, 1), padding=(0, 0))
+
+        # initialize weights
+        self._initialize_weights()
+
+    def _initialize_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
+                if m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.constant_(m.bias, 0)
+
+        # initialize final conv weights to 0: 流行りのzero conv
+        nn.init.constant_(self.conv_final.weight, 0)
+
+    def forward(self, x):
+        inp = x
+
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu1(x)
+
+        # いくつかのresblockを通した後に、residualを足すことで精度向上と学習速度向上が見込めるはず
+        residual = x
+        x = self.resblock1(x)
+        x = self.resblock2(x)
+        x = self.resblock3(x)
+        x = self.resblock4(x)
+        x = x + residual
+        residual = x
+        x = self.resblock5(x)
+        x = self.resblock6(x)
+        x = self.resblock7(x)
+        x = self.resblock8(x)
+        x = x + residual
+        residual = x
+        x = self.resblock9(x)
+        x = self.resblock10(x)
+        x = self.resblock11(x)
+        x = self.resblock12(x)
+        x = x + residual
+        residual = x
+        x = self.resblock13(x)
+        x = self.resblock14(x)
+        x = self.resblock15(x)
+        x = self.resblock16(x)
+        x = x + residual
+        residual = x
+        x = self.resblock17(x)
+        x = self.resblock18(x)
+        x = self.resblock19(x)
+        x = self.resblock20(x)
+        x = x + residual
+
+        x = self.conv2(x)
+        x = self.bn2(x)
+        x = self.relu2(x)
+        x = self.conv3(x)
+        x = self.bn3(x)
+
+        # ここにreluを入れないほうがいい気がする
+
+        x = self.conv_final(x)
+
+        # network estimates the difference between the input and the output
+        x = x + inp
+
+        return x
+
+    def support_latents(self) -> bool:
+        return False
+
+    def upscale(
+        self,
+        vae: AutoencoderKL,
+        lowreso_images: List[Image.Image],
+        lowreso_latents: torch.Tensor,
+        dtype: torch.dtype,
+        width: int,
+        height: int,
+        batch_size: int = 1,
+        vae_batch_size: int = 1,
+    ):
+        # assertion
+        assert lowreso_images is not None, "Upscaler requires lowreso image"
+
+        # make upsampled image with lanczos4
+        upsampled_images = []
+        for lowreso_image in lowreso_images:
+            upsampled_image = np.array(lowreso_image.resize((width, height), Image.LANCZOS))
+            upsampled_images.append(upsampled_image)
+
+        # convert to tensor: this tensor is too large to be converted to cuda
+        upsampled_images = [torch.from_numpy(upsampled_image).permute(2, 0, 1).float() for upsampled_image in upsampled_images]
+        upsampled_images = torch.stack(upsampled_images, dim=0)
+        upsampled_images = upsampled_images.to(dtype)
+
+        # normalize to [-1, 1]
+        upsampled_images = upsampled_images / 127.5 - 1.0
+
+        # convert upsample images to latents with batch size
+        # print("Encoding upsampled (LANCZOS4) images...")
+        upsampled_latents = []
+        for i in tqdm(range(0, upsampled_images.shape[0], vae_batch_size)):
+            batch = upsampled_images[i : i + vae_batch_size].to(vae.device)
+            with torch.no_grad():
+                batch = vae.encode(batch).latent_dist.sample()
+            upsampled_latents.append(batch)
+
+        upsampled_latents = torch.cat(upsampled_latents, dim=0)
+
+        # upscale (refine) latents with this model with batch size
+        print("Upscaling latents...")
+        upscaled_latents = []
+        for i in range(0, upsampled_latents.shape[0], batch_size):
+            with torch.no_grad():
+                upscaled_latents.append(self.forward(upsampled_latents[i : i + batch_size]))
+        upscaled_latents = torch.cat(upscaled_latents, dim=0)
+
+        return upscaled_latents * 0.18215
+
+
+# external interface: returns a model
+def create_upscaler(**kwargs):
+    weights = kwargs["weights"]
+    model = Upscaler()
+
+    print(f"Loading weights from {weights}...")
+    if os.path.splitext(weights)[1] == ".safetensors":
+        from safetensors.torch import load_file
+
+        sd = load_file(weights)
+    else:
+        sd = torch.load(weights, map_location=torch.device("cpu"))
+    model.load_state_dict(sd)
+    return model
+
+
+# another interface: upscale images with a model for given images from command line
+def upscale_images(args: argparse.Namespace):
+    DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    us_dtype = torch.float16  # TODO: support fp32/bf16
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # load VAE with Diffusers
+    assert args.vae_path is not None, "VAE path is required"
+    print(f"Loading VAE from {args.vae_path}...")
+    vae = AutoencoderKL.from_pretrained(args.vae_path, subfolder="vae")
+    vae.to(DEVICE, dtype=us_dtype)
+
+    # prepare model
+    print("Preparing model...")
+    upscaler: Upscaler = create_upscaler(weights=args.weights)
+    # print("Loading weights from", args.weights)
+    # upscaler.load_state_dict(torch.load(args.weights))
+    upscaler.eval()
+    upscaler.to(DEVICE, dtype=us_dtype)
+
+    # load images
+    image_paths = glob.glob(args.image_pattern)
+    images = []
+    for image_path in image_paths:
+        image = Image.open(image_path)
+        image = image.convert("RGB")
+
+        # make divisible by 8
+        width = image.width
+        height = image.height
+        if width % 8 != 0:
+            width = width - (width % 8)
+        if height % 8 != 0:
+            height = height - (height % 8)
+        if width != image.width or height != image.height:
+            image = image.crop((0, 0, width, height))
+
+        images.append(image)
+
+    # debug output
+    if args.debug:
+        for image, image_path in zip(images, image_paths):
+            image_debug = image.resize((image.width * 2, image.height * 2), Image.LANCZOS)
+
+            basename = os.path.basename(image_path)
+            basename_wo_ext, ext = os.path.splitext(basename)
+            dest_file_name = os.path.join(args.output_dir, f"{basename_wo_ext}_lanczos4{ext}")
+            image_debug.save(dest_file_name)
+
+    # upscale
+    print("Upscaling...")
+    upscaled_latents = upscaler.upscale(
+        vae, images, None, us_dtype, width * 2, height * 2, batch_size=args.batch_size, vae_batch_size=args.vae_batch_size
+    )
+    upscaled_latents /= 0.18215
+
+    # decode with batch
+    print("Decoding...")
+    upscaled_images = []
+    for i in tqdm(range(0, upscaled_latents.shape[0], args.vae_batch_size)):
+        with torch.no_grad():
+            batch = vae.decode(upscaled_latents[i : i + args.vae_batch_size]).sample
+        batch = batch.to("cpu")
+        upscaled_images.append(batch)
+    upscaled_images = torch.cat(upscaled_images, dim=0)
+
+    # tensor to numpy
+    upscaled_images = upscaled_images.permute(0, 2, 3, 1).numpy()
+    upscaled_images = (upscaled_images + 1.0) * 127.5
+    upscaled_images = upscaled_images.clip(0, 255).astype(np.uint8)
+
+    upscaled_images = upscaled_images[..., ::-1]
+
+    # save images
+    for i, image in enumerate(upscaled_images):
+        basename = os.path.basename(image_paths[i])
+        basename_wo_ext, ext = os.path.splitext(basename)
+        dest_file_name = os.path.join(args.output_dir, f"{basename_wo_ext}_upscaled{ext}")
+        cv2.imwrite(dest_file_name, image)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vae_path", type=str, default=None, help="VAE path")
+    parser.add_argument("--weights", type=str, default=None, help="Weights path")
+    parser.add_argument("--image_pattern", type=str, default=None, help="Image pattern")
+    parser.add_argument("--output_dir", type=str, default=".", help="Output directory")
+    parser.add_argument("--batch_size", type=int, default=4, help="Batch size")
+    parser.add_argument("--vae_batch_size", type=int, default=1, help="VAE batch size")
+    parser.add_argument("--debug", action="store_true", help="Debug mode")
+
+    args = parser.parse_args()
+    upscale_images(args)
diff --git a/tools/lycoris_locon_extract.py b/tools/lycoris_locon_extract.py
new file mode 100644
index 0000000000000000000000000000000000000000..75b55490b16b684ff2abe76e989831188483b1f8
--- /dev/null
+++ b/tools/lycoris_locon_extract.py
@@ -0,0 +1,129 @@
+import os, sys
+sys.path.insert(0, os.getcwd())
+import argparse
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "base_model", help="The model which use it to train the dreambooth model",
+        default='', type=str
+    )
+    parser.add_argument(
+        "db_model", help="the dreambooth model you want to extract the locon",
+        default='', type=str
+    )
+    parser.add_argument(
+        "output_name", help="the output model",
+        default='./out.pt', type=str
+    )
+    parser.add_argument(
+        "--is_v2", help="Your base/db model is sd v2 or not",
+        default=False, action="store_true"
+    )
+    parser.add_argument(
+        "--device", help="Which device you want to use to extract the locon",
+        default='cpu', type=str
+    )
+    parser.add_argument(
+        "--mode", 
+        help=(
+            'extraction mode, can be "fixed", "threshold", "ratio", "quantile". '
+            'If not "fixed", network_dim and conv_dim will be ignored'
+        ),
+        default='fixed', type=str
+    )
+    parser.add_argument(
+        "--safetensors", help='use safetensors to save locon model',
+        default=False, action="store_true"
+    )
+    parser.add_argument(
+        "--linear_dim", help="network dim for linear layer in fixed mode",
+        default=1, type=int
+    )
+    parser.add_argument(
+        "--conv_dim", help="network dim for conv layer in fixed mode",
+        default=1, type=int
+    )
+    parser.add_argument(
+        "--linear_threshold", help="singular value threshold for linear layer in threshold mode",
+        default=0., type=float
+    )
+    parser.add_argument(
+        "--conv_threshold", help="singular value threshold for conv layer in threshold mode",
+        default=0., type=float
+    )
+    parser.add_argument(
+        "--linear_ratio", help="singular ratio for linear layer in ratio mode",
+        default=0., type=float
+    )
+    parser.add_argument(
+        "--conv_ratio", help="singular ratio for conv layer in ratio mode",
+        default=0., type=float
+    )
+    parser.add_argument(
+        "--linear_quantile", help="singular value quantile for linear layer quantile mode",
+        default=1., type=float
+    )
+    parser.add_argument(
+        "--conv_quantile", help="singular value quantile for conv layer quantile mode",
+        default=1., type=float
+    )
+    parser.add_argument(
+        "--use_sparse_bias", help="enable sparse bias",
+        default=False, action="store_true"
+    )
+    parser.add_argument(
+        "--sparsity", help="sparsity for sparse bias",
+        default=0.98, type=float
+    )
+    parser.add_argument(
+        "--disable_cp", help="don't use cp decomposition",
+        default=False, action="store_true"
+    )
+    return parser.parse_args()
+ARGS = get_args()
+
+
+from lycoris.utils import extract_diff
+from lycoris.kohya_model_utils import load_models_from_stable_diffusion_checkpoint
+
+import torch
+from safetensors.torch import save_file
+
+
+def main():
+    args = ARGS
+    base = load_models_from_stable_diffusion_checkpoint(args.is_v2, args.base_model)
+    db = load_models_from_stable_diffusion_checkpoint(args.is_v2, args.db_model)
+    
+    linear_mode_param = {
+        'fixed': args.linear_dim,
+        'threshold': args.linear_threshold,
+        'ratio': args.linear_ratio,
+        'quantile': args.linear_quantile,
+    }[args.mode]
+    conv_mode_param = {
+        'fixed': args.conv_dim,
+        'threshold': args.conv_threshold,
+        'ratio': args.conv_ratio,
+        'quantile': args.conv_quantile,
+    }[args.mode]
+    
+    state_dict = extract_diff(
+        base, db,
+        args.mode,
+        linear_mode_param, conv_mode_param,
+        args.device, 
+        args.use_sparse_bias, args.sparsity,
+        not args.disable_cp
+    )
+    
+    if args.safetensors:
+        save_file(state_dict, args.output_name)
+    else:
+        torch.save(state_dict, args.output_name)
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/tools/lycoris_utils.py b/tools/lycoris_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..25b066b5c8132c65904a8fadf21417a1a2d3f09f
--- /dev/null
+++ b/tools/lycoris_utils.py
@@ -0,0 +1,504 @@
+from typing import *
+
+import numpy as np
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import torch.linalg as linalg
+
+from tqdm import tqdm
+
+
+def make_sparse(t: torch.Tensor, sparsity=0.95):
+    abs_t = torch.abs(t)
+    np_array = abs_t.detach().cpu().numpy()
+    quan = float(np.quantile(np_array, sparsity))
+    sparse_t = t.masked_fill(abs_t < quan, 0)
+    return sparse_t
+
+
+def extract_conv(
+    weight: Union[torch.Tensor, nn.Parameter],
+    mode = 'fixed',
+    mode_param = 0,
+    device = 'cpu',
+    is_cp = False,
+) -> Tuple[nn.Parameter, nn.Parameter]:
+    weight = weight.to(device)
+    out_ch, in_ch, kernel_size, _ = weight.shape
+    
+    U, S, Vh = linalg.svd(weight.reshape(out_ch, -1))
+    
+    if mode=='fixed':
+        lora_rank = mode_param
+    elif mode=='threshold':
+        assert mode_param>=0
+        lora_rank = torch.sum(S>mode_param)
+    elif mode=='ratio':
+        assert 1>=mode_param>=0
+        min_s = torch.max(S)*mode_param
+        lora_rank = torch.sum(S>min_s)
+    elif mode=='quantile' or mode=='percentile':
+        assert 1>=mode_param>=0
+        s_cum = torch.cumsum(S, dim=0)
+        min_cum_sum = mode_param * torch.sum(S)
+        lora_rank = torch.sum(s_cum<min_cum_sum)
+    else:
+        raise NotImplementedError('Extract mode should be "fixed", "threshold", "ratio" or "quantile"')
+    lora_rank = max(1, lora_rank)
+    lora_rank = min(out_ch, in_ch, lora_rank)
+    if lora_rank>=out_ch/2 and not is_cp:
+        return weight, 'full'
+    
+    U = U[:, :lora_rank]
+    S = S[:lora_rank]
+    U = U @ torch.diag(S)
+    Vh = Vh[:lora_rank, :]
+    
+    diff = (weight - (U @ Vh).reshape(out_ch, in_ch, kernel_size, kernel_size)).detach()
+    extract_weight_A = Vh.reshape(lora_rank, in_ch, kernel_size, kernel_size).detach()
+    extract_weight_B = U.reshape(out_ch, lora_rank, 1, 1).detach()
+    del U, S, Vh, weight
+    return (extract_weight_A, extract_weight_B, diff), 'low rank'
+
+
+def extract_linear(
+    weight: Union[torch.Tensor, nn.Parameter],
+    mode = 'fixed',
+    mode_param = 0,
+    device = 'cpu',
+) -> Tuple[nn.Parameter, nn.Parameter]:
+    weight = weight.to(device)
+    out_ch, in_ch = weight.shape
+    
+    U, S, Vh = linalg.svd(weight)
+    
+    if mode=='fixed':
+        lora_rank = mode_param
+    elif mode=='threshold':
+        assert mode_param>=0
+        lora_rank = torch.sum(S>mode_param)
+    elif mode=='ratio':
+        assert 1>=mode_param>=0
+        min_s = torch.max(S)*mode_param
+        lora_rank = torch.sum(S>min_s)
+    elif mode=='quantile' or mode=='percentile':
+        assert 1>=mode_param>=0
+        s_cum = torch.cumsum(S, dim=0)
+        min_cum_sum = mode_param * torch.sum(S)
+        lora_rank = torch.sum(s_cum<min_cum_sum)
+    else:
+        raise NotImplementedError('Extract mode should be "fixed", "threshold", "ratio" or "quantile"')
+    lora_rank = max(1, lora_rank)
+    lora_rank = min(out_ch, in_ch, lora_rank)
+    if lora_rank>=out_ch/2:
+        return weight, 'full'
+    
+    U = U[:, :lora_rank]
+    S = S[:lora_rank]
+    U = U @ torch.diag(S)
+    Vh = Vh[:lora_rank, :]
+    
+    diff = (weight - U @ Vh).detach()
+    extract_weight_A = Vh.reshape(lora_rank, in_ch).detach()
+    extract_weight_B = U.reshape(out_ch, lora_rank).detach()
+    del U, S, Vh, weight
+    return (extract_weight_A, extract_weight_B, diff), 'low rank'
+
+
+def extract_diff(
+    base_model,
+    db_model,
+    mode = 'fixed',
+    linear_mode_param = 0,
+    conv_mode_param = 0,
+    extract_device = 'cpu',
+    use_bias = False,
+    sparsity = 0.98,
+    small_conv = True
+):
+    UNET_TARGET_REPLACE_MODULE = [
+        "Transformer2DModel", 
+        "Attention", 
+        "ResnetBlock2D", 
+        "Downsample2D", 
+        "Upsample2D"
+    ]
+    UNET_TARGET_REPLACE_NAME = [
+        "conv_in",
+        "conv_out",
+        "time_embedding.linear_1",
+        "time_embedding.linear_2",
+    ]
+    TEXT_ENCODER_TARGET_REPLACE_MODULE = ["CLIPAttention", "CLIPMLP"]
+    LORA_PREFIX_UNET = 'lora_unet'
+    LORA_PREFIX_TEXT_ENCODER = 'lora_te'
+    def make_state_dict(
+        prefix, 
+        root_module: torch.nn.Module,
+        target_module: torch.nn.Module,
+        target_replace_modules,
+        target_replace_names = []
+    ):
+        loras = {}
+        temp = {}
+        temp_name = {}
+        
+        for name, module in root_module.named_modules():
+            if module.__class__.__name__ in target_replace_modules:
+                temp[name] = {}
+                for child_name, child_module in module.named_modules():
+                    if child_module.__class__.__name__ not in {'Linear', 'Conv2d'}:
+                        continue
+                    temp[name][child_name] = child_module.weight
+            elif name in target_replace_names:
+                temp_name[name] = module.weight
+        
+        for name, module in tqdm(list(target_module.named_modules())):
+            if name in temp:
+                weights = temp[name]
+                for child_name, child_module in module.named_modules():
+                    lora_name = prefix + '.' + name + '.' + child_name
+                    lora_name = lora_name.replace('.', '_')
+                    layer = child_module.__class__.__name__
+                    if layer in {'Linear', 'Conv2d'}:
+                        root_weight = child_module.weight
+                        if torch.allclose(root_weight, weights[child_name]):
+                            continue
+                    
+                    if layer == 'Linear':
+                        weight, decompose_mode = extract_linear(
+                            (child_module.weight - weights[child_name]),
+                            mode,
+                            linear_mode_param,
+                            device = extract_device,
+                        )
+                        if decompose_mode == 'low rank':
+                            extract_a, extract_b, diff = weight
+                    elif layer == 'Conv2d':
+                        is_linear = (child_module.weight.shape[2] == 1
+                                     and child_module.weight.shape[3] == 1)
+                        weight, decompose_mode = extract_conv(
+                            (child_module.weight - weights[child_name]), 
+                            mode,
+                            linear_mode_param if is_linear else conv_mode_param,
+                            device = extract_device,
+                        )
+                        if decompose_mode == 'low rank':
+                            extract_a, extract_b, diff = weight
+                        if small_conv and not is_linear and decompose_mode == 'low rank':
+                            dim = extract_a.size(0)
+                            (extract_c, extract_a, _), _ = extract_conv(
+                                extract_a.transpose(0, 1), 
+                                'fixed', dim, 
+                                extract_device, True
+                            )
+                            extract_a = extract_a.transpose(0, 1)
+                            extract_c = extract_c.transpose(0, 1)
+                            loras[f'{lora_name}.lora_mid.weight'] = extract_c.detach().cpu().contiguous().half()
+                            diff = child_module.weight - torch.einsum(
+                                'i j k l, j r, p i -> p r k l', 
+                                extract_c, extract_a.flatten(1, -1), extract_b.flatten(1, -1)
+                            ).detach().cpu().contiguous()
+                            del extract_c
+                    else:
+                        continue
+                    if decompose_mode == 'low rank':
+                        loras[f'{lora_name}.lora_down.weight'] = extract_a.detach().cpu().contiguous().half()
+                        loras[f'{lora_name}.lora_up.weight'] = extract_b.detach().cpu().contiguous().half()
+                        loras[f'{lora_name}.alpha'] = torch.Tensor([extract_a.shape[0]]).half()
+                        if use_bias:
+                            diff = diff.detach().cpu().reshape(extract_b.size(0), -1)
+                            sparse_diff = make_sparse(diff, sparsity).to_sparse().coalesce()
+                            
+                            indices = sparse_diff.indices().to(torch.int16)
+                            values = sparse_diff.values().half()
+                            loras[f'{lora_name}.bias_indices'] = indices
+                            loras[f'{lora_name}.bias_values'] = values
+                            loras[f'{lora_name}.bias_size'] = torch.tensor(diff.shape).to(torch.int16)
+                        del extract_a, extract_b, diff
+                    elif decompose_mode == 'full':
+                        loras[f'{lora_name}.diff'] = weight.detach().cpu().contiguous().half()
+                    else:
+                        raise NotImplementedError
+            elif name in temp_name:
+                weights = temp_name[name]
+                lora_name = prefix + '.' + name
+                lora_name = lora_name.replace('.', '_')
+                layer = module.__class__.__name__
+                
+                if layer in {'Linear', 'Conv2d'}:
+                    root_weight = module.weight
+                    if torch.allclose(root_weight, weights):
+                        continue
+                
+                if layer == 'Linear':
+                    weight, decompose_mode = extract_linear(
+                        (root_weight - weights),
+                        mode,
+                        linear_mode_param,
+                        device = extract_device,
+                    )
+                    if decompose_mode == 'low rank':
+                        extract_a, extract_b, diff = weight
+                elif layer == 'Conv2d':
+                    is_linear = (
+                        root_weight.shape[2] == 1
+                        and root_weight.shape[3] == 1
+                    )
+                    weight, decompose_mode = extract_conv(
+                        (root_weight - weights), 
+                        mode,
+                        linear_mode_param if is_linear else conv_mode_param,
+                        device = extract_device,
+                    )
+                    if decompose_mode == 'low rank':
+                        extract_a, extract_b, diff = weight
+                    if small_conv and not is_linear and decompose_mode == 'low rank':
+                        dim = extract_a.size(0)
+                        (extract_c, extract_a, _), _ = extract_conv(
+                            extract_a.transpose(0, 1), 
+                            'fixed', dim, 
+                            extract_device, True
+                        )
+                        extract_a = extract_a.transpose(0, 1)
+                        extract_c = extract_c.transpose(0, 1)
+                        loras[f'{lora_name}.lora_mid.weight'] = extract_c.detach().cpu().contiguous().half()
+                        diff = root_weight - torch.einsum(
+                            'i j k l, j r, p i -> p r k l', 
+                            extract_c, extract_a.flatten(1, -1), extract_b.flatten(1, -1)
+                        ).detach().cpu().contiguous()
+                        del extract_c
+                else:
+                    continue
+                if decompose_mode == 'low rank':
+                    loras[f'{lora_name}.lora_down.weight'] = extract_a.detach().cpu().contiguous().half()
+                    loras[f'{lora_name}.lora_up.weight'] = extract_b.detach().cpu().contiguous().half()
+                    loras[f'{lora_name}.alpha'] = torch.Tensor([extract_a.shape[0]]).half()
+                    if use_bias:
+                        diff = diff.detach().cpu().reshape(extract_b.size(0), -1)
+                        sparse_diff = make_sparse(diff, sparsity).to_sparse().coalesce()
+                        
+                        indices = sparse_diff.indices().to(torch.int16)
+                        values = sparse_diff.values().half()
+                        loras[f'{lora_name}.bias_indices'] = indices
+                        loras[f'{lora_name}.bias_values'] = values
+                        loras[f'{lora_name}.bias_size'] = torch.tensor(diff.shape).to(torch.int16)
+                    del extract_a, extract_b, diff
+                elif decompose_mode == 'full':
+                    loras[f'{lora_name}.diff'] = weight.detach().cpu().contiguous().half()
+                else:
+                    raise NotImplementedError
+        return loras
+    
+    text_encoder_loras = make_state_dict(
+        LORA_PREFIX_TEXT_ENCODER, 
+        base_model[0], db_model[0], 
+        TEXT_ENCODER_TARGET_REPLACE_MODULE
+    )
+    
+    unet_loras = make_state_dict(
+        LORA_PREFIX_UNET,
+        base_model[2], db_model[2], 
+        UNET_TARGET_REPLACE_MODULE,
+        UNET_TARGET_REPLACE_NAME
+    )
+    print(len(text_encoder_loras), len(unet_loras))
+    return text_encoder_loras|unet_loras
+
+
+def get_module(
+    lyco_state_dict: Dict,
+    lora_name
+):
+    if f'{lora_name}.lora_up.weight' in lyco_state_dict:
+        up = lyco_state_dict[f'{lora_name}.lora_up.weight']
+        down = lyco_state_dict[f'{lora_name}.lora_down.weight']
+        mid = lyco_state_dict.get(f'{lora_name}.lora_mid.weight', None)
+        alpha = lyco_state_dict.get(f'{lora_name}.alpha', None)
+        return 'locon', (up, down, mid, alpha)
+    elif f'{lora_name}.hada_w1_a' in lyco_state_dict:
+        w1a = lyco_state_dict[f'{lora_name}.hada_w1_a']
+        w1b = lyco_state_dict[f'{lora_name}.hada_w1_b']
+        w2a = lyco_state_dict[f'{lora_name}.hada_w2_a']
+        w2b = lyco_state_dict[f'{lora_name}.hada_w2_b']
+        t1 = lyco_state_dict.get(f'{lora_name}.hada_t1', None)
+        t2 = lyco_state_dict.get(f'{lora_name}.hada_t2', None)
+        alpha = lyco_state_dict.get(f'{lora_name}.alpha', None)
+        return 'hada', (w1a, w1b, w2a, w2b, t1, t2, alpha)
+    elif f'{lora_name}.weight' in lyco_state_dict:
+        weight = lyco_state_dict[f'{lora_name}.weight']
+        on_input = lyco_state_dict.get(f'{lora_name}.on_input', False)
+        return 'ia3', (weight, on_input)
+    elif (f'{lora_name}.lokr_w1' in lyco_state_dict
+          or f'{lora_name}.lokr_w1_a' in lyco_state_dict):
+        w1 = lyco_state_dict.get(f'{lora_name}.lokr_w1', None)
+        w1a = lyco_state_dict.get(f'{lora_name}.lokr_w1_a', None)
+        w1b = lyco_state_dict.get(f'{lora_name}.lokr_w1_b', None)
+        w2 = lyco_state_dict.get(f'{lora_name}.lokr_w2', None)
+        w2a = lyco_state_dict.get(f'{lora_name}.lokr_w2_a', None)
+        w2b = lyco_state_dict.get(f'{lora_name}.lokr_w2_b', None)
+        t1 = lyco_state_dict.get(f'{lora_name}.lokr_t1', None)
+        t2 = lyco_state_dict.get(f'{lora_name}.lokr_t2', None)
+        alpha = lyco_state_dict.get(f'{lora_name}.alpha', None)
+        return 'kron', (w1, w1a, w1b, w2, w2a, w2b, t1, t2, alpha)
+    elif f'{lora_name}.diff' in lyco_state_dict:
+        return 'full', lyco_state_dict[f'{lora_name}.diff']
+    else:
+        return 'None', ()
+
+
+def cp_weight_from_conv(
+    up, down, mid
+):
+    up = up.reshape(up.size(0), up.size(1))
+    down = down.reshape(down.size(0), down.size(1))
+    return torch.einsum('m n w h, i m, n j -> i j w h', mid, up, down)
+
+def cp_weight(
+    wa, wb, t
+):
+    temp = torch.einsum('i j k l, j r -> i r k l', t, wb)
+    return torch.einsum('i j k l, i r -> r j k l', temp, wa)
+
+
+@torch.no_grad()
+def rebuild_weight(module_type, params, orig_weight, scale=1):
+    if orig_weight is None:
+        return orig_weight
+    merged = orig_weight
+    if module_type == 'locon':
+        up, down, mid, alpha = params
+        if alpha is not None:
+            scale *= alpha/up.size(1)
+        if mid is not None:
+            rebuild = cp_weight_from_conv(up, down, mid)
+        else:
+            rebuild = up.reshape(up.size(0),-1) @ down.reshape(down.size(0), -1)
+        merged = orig_weight + rebuild.reshape(orig_weight.shape) * scale
+        del up, down, mid, alpha, params, rebuild
+    elif module_type == 'hada':
+        w1a, w1b, w2a, w2b, t1, t2, alpha = params
+        if alpha is not None:
+            scale *= alpha / w1b.size(0)
+        if t1 is not None:
+            rebuild1 = cp_weight(w1a, w1b, t1)
+        else:
+            rebuild1 = w1a @ w1b
+        if t2 is not None:
+            rebuild2 = cp_weight(w2a, w2b, t2)
+        else:
+            rebuild2 = w2a @ w2b
+        rebuild = (rebuild1 * rebuild2).reshape(orig_weight.shape)
+        merged = orig_weight + rebuild * scale
+        del w1a, w1b, w2a, w2b, t1, t2, alpha, params, rebuild, rebuild1, rebuild2
+    elif module_type == 'ia3':
+        weight, on_input = params
+        if not on_input:
+            weight = weight.reshape(-1, 1)
+        merged = orig_weight + weight * orig_weight * scale
+        del weight, on_input, params
+    elif module_type == 'kron':
+        w1, w1a, w1b, w2, w2a, w2b, t1, t2, alpha = params
+        if alpha is not None and (w1b is not None or w2b is not None):
+            scale *= alpha / (w1b.size(0) if w1b else w2b.size(0))
+        if w1a is not None and w1b is not None:
+            if t1:
+                w1 = cp_weight(w1a, w1b, t1)
+            else:
+                w1 = w1a @ w1b
+        if w2a is not None and w2b is not None:
+            if t2:
+                w2 = cp_weight(w2a, w2b, t2)
+            else:
+                w2 = w2a @ w2b
+        rebuild = torch.kron(w1, w2).reshape(orig_weight.shape) 
+        merged = orig_weight + rebuild* scale
+        del w1, w1a, w1b, w2, w2a, w2b, t1, t2, alpha, params, rebuild
+    elif module_type == 'full':
+        rebuild = params.reshape(orig_weight.shape) 
+        merged = orig_weight + rebuild * scale
+        del params, rebuild
+    
+    return merged
+
+
+def merge(
+    base_model,
+    lyco_state_dict,
+    scale: float = 1.0,
+    device = 'cpu'
+):
+    UNET_TARGET_REPLACE_MODULE = [
+        "Transformer2DModel", 
+        "Attention", 
+        "ResnetBlock2D", 
+        "Downsample2D", 
+        "Upsample2D"
+    ]
+    UNET_TARGET_REPLACE_NAME = [
+        "conv_in",
+        "conv_out",
+        "time_embedding.linear_1",
+        "time_embedding.linear_2",
+    ]
+    TEXT_ENCODER_TARGET_REPLACE_MODULE = ["CLIPAttention", "CLIPMLP"]
+    LORA_PREFIX_UNET = 'lora_unet'
+    LORA_PREFIX_TEXT_ENCODER = 'lora_te'
+    merged = 0
+    def merge_state_dict(
+        prefix, 
+        root_module: torch.nn.Module,
+        lyco_state_dict: Dict[str,torch.Tensor],
+        target_replace_modules,
+        target_replace_names = []
+    ):
+        nonlocal merged
+        for name, module in tqdm(list(root_module.named_modules()), desc=f'Merging {prefix}'):
+            if module.__class__.__name__ in target_replace_modules:
+                for child_name, child_module in module.named_modules():
+                    if child_module.__class__.__name__ not in {'Linear', 'Conv2d'}:
+                        continue
+                    lora_name = prefix + '.' + name + '.' + child_name
+                    lora_name = lora_name.replace('.', '_')
+                    
+                    result = rebuild_weight(*get_module(
+                        lyco_state_dict, lora_name
+                    ), getattr(child_module, 'weight'), scale)
+                    if result is not None:
+                        merged += 1
+                        child_module.requires_grad_(False)
+                        child_module.weight.copy_(result)
+            elif name in target_replace_names:
+                lora_name = prefix + '.' + name
+                lora_name = lora_name.replace('.', '_')
+                    
+                result = rebuild_weight(*get_module(
+                    lyco_state_dict, lora_name
+                ), getattr(module, 'weight'), scale)
+                if result is not None:
+                    merged += 1
+                    module.requires_grad_(False)
+                    module.weight.copy_(result)
+    
+    if device == 'cpu':
+        for k, v in tqdm(list(lyco_state_dict.items()), desc='Converting Dtype'):
+            lyco_state_dict[k] = v.float()
+    
+    merge_state_dict(
+        LORA_PREFIX_TEXT_ENCODER,
+        base_model[0],
+        lyco_state_dict,
+        TEXT_ENCODER_TARGET_REPLACE_MODULE,
+        UNET_TARGET_REPLACE_NAME
+    )
+    merge_state_dict(
+        LORA_PREFIX_UNET,
+        base_model[2],
+        lyco_state_dict,
+        UNET_TARGET_REPLACE_MODULE,
+        UNET_TARGET_REPLACE_NAME
+    )
+    print(f'{merged} Modules been merged')
\ No newline at end of file
diff --git a/tools/merge_lycoris.py b/tools/merge_lycoris.py
new file mode 100644
index 0000000000000000000000000000000000000000..92223ca08addd795b8b20c5c549b710149fc3507
--- /dev/null
+++ b/tools/merge_lycoris.py
@@ -0,0 +1,85 @@
+import os, sys
+sys.path.insert(0, os.getcwd())
+import argparse
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "base_model", help="The model you want to merge with loha",
+        default='', type=str
+    )
+    parser.add_argument(
+        "lycoris_model", help="the lyco model you want to merge into sd model",
+        default='', type=str
+    )
+    parser.add_argument(
+        "output_name", help="the output model",
+        default='./out.pt', type=str
+    )
+    parser.add_argument(
+        "--is_v2", help="Your base model is sd v2 or not",
+        default=False, action="store_true"
+    )
+    parser.add_argument(
+        "--device", help="Which device you want to use to merge the weight",
+        default='cpu', type=str
+    )
+    parser.add_argument(
+        "--dtype", help='dtype to save',
+        default='float', type=str
+    )
+    parser.add_argument(
+        "--weight", help='weight for the lyco model to merge',
+        default='1.0', type=float
+    )
+    return parser.parse_args()
+ARGS = get_args()
+
+
+from lycoris_utils import merge
+from lycoris.kohya_model_utils import (
+    load_models_from_stable_diffusion_checkpoint,
+    save_stable_diffusion_checkpoint,
+    load_file
+)
+
+import torch
+
+
+def main():
+    base = load_models_from_stable_diffusion_checkpoint(ARGS.is_v2, ARGS.base_model)
+    if ARGS.lycoris_model.rsplit('.', 1)[-1] == 'safetensors':
+        lyco = load_file(ARGS.lycoris_model)
+    else:
+        lyco = torch.load(ARGS.lycoris_model)
+    
+    dtype_str = ARGS.dtype.replace('fp', 'float').replace('bf', 'bfloat')
+    dtype = {
+        'float': torch.float,
+        'float16': torch.float16,
+        'float32': torch.float32,
+        'float64': torch.float64,
+        'bfloat': torch.bfloat16,
+        'bfloat16': torch.bfloat16,
+    }.get(dtype_str, None)
+    if dtype is None:
+        raise ValueError(f'Cannot Find the dtype "{dtype}"')
+    
+    merge(
+        base,
+        lyco,
+        ARGS.weight,
+        ARGS.device
+    )
+    
+    save_stable_diffusion_checkpoint(
+        ARGS.is_v2, ARGS.output_name, 
+        base[0], base[2], 
+        None, 0, 0, dtype, 
+        base[1]
+    )
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/tools/original_control_net.py b/tools/original_control_net.py
new file mode 100644
index 0000000000000000000000000000000000000000..582794de71860f7c32ec80fe4da6775db63cf681
--- /dev/null
+++ b/tools/original_control_net.py
@@ -0,0 +1,321 @@
+from typing import List, NamedTuple, Any
+import numpy as np
+import cv2
+import torch
+from safetensors.torch import load_file
+
+from diffusers import UNet2DConditionModel
+from diffusers.models.unet_2d_condition import UNet2DConditionOutput
+
+import library.model_util as model_util
+
+
+class ControlNetInfo(NamedTuple):
+  unet: Any
+  net: Any
+  prep: Any
+  weight: float
+  ratio: float
+
+
+class ControlNet(torch.nn.Module):
+  def __init__(self) -> None:
+    super().__init__()
+
+    # make control model
+    self.control_model = torch.nn.Module()
+
+    dims = [320, 320, 320, 320, 640, 640, 640, 1280, 1280, 1280, 1280, 1280]
+    zero_convs = torch.nn.ModuleList()
+    for i, dim in enumerate(dims):
+      sub_list = torch.nn.ModuleList([torch.nn.Conv2d(dim, dim, 1)])
+      zero_convs.append(sub_list)
+    self.control_model.add_module("zero_convs", zero_convs)
+
+    middle_block_out = torch.nn.Conv2d(1280, 1280, 1)
+    self.control_model.add_module("middle_block_out", torch.nn.ModuleList([middle_block_out]))
+
+    dims = [16, 16, 32, 32, 96, 96, 256, 320]
+    strides = [1, 1, 2, 1, 2, 1, 2, 1]
+    prev_dim = 3
+    input_hint_block = torch.nn.Sequential()
+    for i, (dim, stride) in enumerate(zip(dims, strides)):
+      input_hint_block.append(torch.nn.Conv2d(prev_dim, dim, 3, stride, 1))
+      if i < len(dims) - 1:
+        input_hint_block.append(torch.nn.SiLU())
+      prev_dim = dim
+    self.control_model.add_module("input_hint_block", input_hint_block)
+
+
+def load_control_net(v2, unet, model):
+  device = unet.device
+
+  # control sdからキー変換しつつU-Netに対応する部分のみ取り出し、DiffusersのU-Netに読み込む
+  # state dictを読み込む
+  print(f"ControlNet: loading control SD model : {model}")
+
+  if model_util.is_safetensors(model):
+    ctrl_sd_sd = load_file(model)
+  else:
+    ctrl_sd_sd = torch.load(model, map_location='cpu')
+    ctrl_sd_sd = ctrl_sd_sd.pop("state_dict", ctrl_sd_sd)
+
+  # 重みをU-Netに読み込めるようにする。ControlNetはSD版のstate dictなので、それを読み込む
+  is_difference = "difference" in ctrl_sd_sd
+  print("ControlNet: loading difference:", is_difference)
+
+  # ControlNetには存在しないキーがあるので、まず現在のU-NetでSD版の全keyを作っておく
+  # またTransfer Controlの元weightとなる
+  ctrl_unet_sd_sd = model_util.convert_unet_state_dict_to_sd(v2, unet.state_dict())
+
+  # 元のU-Netに影響しないようにコピーする。またprefixが付いていないので付ける
+  for key in list(ctrl_unet_sd_sd.keys()):
+    ctrl_unet_sd_sd["model.diffusion_model." + key] = ctrl_unet_sd_sd.pop(key).clone()
+
+  zero_conv_sd = {}
+  for key in list(ctrl_sd_sd.keys()):
+    if key.startswith("control_"):
+      unet_key = "model.diffusion_" + key[len("control_"):]
+      if unet_key not in ctrl_unet_sd_sd:               # zero conv
+        zero_conv_sd[key] = ctrl_sd_sd[key]
+        continue
+      if is_difference:                                 # Transfer Control
+        ctrl_unet_sd_sd[unet_key] += ctrl_sd_sd[key].to(device, dtype=unet.dtype)
+      else:
+        ctrl_unet_sd_sd[unet_key] = ctrl_sd_sd[key].to(device, dtype=unet.dtype)
+
+  unet_config = model_util.create_unet_diffusers_config(v2)
+  ctrl_unet_du_sd = model_util.convert_ldm_unet_checkpoint(v2, ctrl_unet_sd_sd, unet_config)    # DiffUsers版ControlNetのstate dict
+
+  # ControlNetのU-Netを作成する
+  ctrl_unet = UNet2DConditionModel(**unet_config)
+  info = ctrl_unet.load_state_dict(ctrl_unet_du_sd)
+  print("ControlNet: loading Control U-Net:", info)
+
+  # U-Net以外のControlNetを作成する
+  # TODO support middle only
+  ctrl_net = ControlNet()
+  info = ctrl_net.load_state_dict(zero_conv_sd)
+  print("ControlNet: loading ControlNet:", info)
+
+  ctrl_unet.to(unet.device, dtype=unet.dtype)
+  ctrl_net.to(unet.device, dtype=unet.dtype)
+  return ctrl_unet, ctrl_net
+
+
+def load_preprocess(prep_type: str):
+  if prep_type is None or prep_type.lower() == "none":
+    return None
+
+  if prep_type.startswith("canny"):
+    args = prep_type.split("_")
+    th1 = int(args[1]) if len(args) >= 2 else 63
+    th2 = int(args[2]) if len(args) >= 3 else 191
+
+    def canny(img):
+      img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+      return cv2.Canny(img, th1, th2)
+    return canny
+
+  print("Unsupported prep type:", prep_type)
+  return None
+
+
+def preprocess_ctrl_net_hint_image(image):
+  image = np.array(image).astype(np.float32) / 255.0
+  # ControlNetのサンプルはcv2を使っているが、読み込みはGradioなので実はRGBになっている
+  # image = image[:, :, ::-1].copy()                         # rgb to bgr
+  image = image[None].transpose(0, 3, 1, 2)       # nchw
+  image = torch.from_numpy(image)
+  return image                              # 0 to 1
+
+
+def get_guided_hints(control_nets: List[ControlNetInfo], num_latent_input, b_size, hints):
+  guided_hints = []
+  for i, cnet_info in enumerate(control_nets):
+    # hintは 1枚目の画像のcnet1, 1枚目の画像のcnet2, 1枚目の画像のcnet3, 2枚目の画像のcnet1, 2枚目の画像のcnet2 ... と並んでいること
+    b_hints = []
+    if len(hints) == 1:           # すべて同じ画像をhintとして使う
+      hint = hints[0]
+      if cnet_info.prep is not None:
+        hint = cnet_info.prep(hint)
+      hint = preprocess_ctrl_net_hint_image(hint)
+      b_hints = [hint for _ in range(b_size)]
+    else:
+      for bi in range(b_size):
+        hint = hints[(bi * len(control_nets) + i) % len(hints)]
+        if cnet_info.prep is not None:
+          hint = cnet_info.prep(hint)
+        hint = preprocess_ctrl_net_hint_image(hint)
+        b_hints.append(hint)
+    b_hints = torch.cat(b_hints, dim=0)
+    b_hints = b_hints.to(cnet_info.unet.device, dtype=cnet_info.unet.dtype)
+
+    guided_hint = cnet_info.net.control_model.input_hint_block(b_hints)
+    guided_hints.append(guided_hint)
+  return guided_hints
+
+
+def call_unet_and_control_net(step, num_latent_input, original_unet, control_nets: List[ControlNetInfo], guided_hints, current_ratio, sample, timestep, encoder_hidden_states):
+  # ControlNet
+  # 複数のControlNetの場合は、出力をマージするのではなく交互に適用する
+  cnet_cnt = len(control_nets)
+  cnet_idx = step % cnet_cnt
+  cnet_info = control_nets[cnet_idx]
+
+  # print(current_ratio, cnet_info.prep, cnet_info.weight, cnet_info.ratio)
+  if cnet_info.ratio < current_ratio:
+    return original_unet(sample, timestep, encoder_hidden_states)
+
+  guided_hint = guided_hints[cnet_idx]
+  guided_hint = guided_hint.repeat((num_latent_input, 1, 1, 1))
+  outs = unet_forward(True, cnet_info.net, cnet_info.unet, guided_hint, None, sample, timestep, encoder_hidden_states)
+  outs = [o * cnet_info.weight for o in outs]
+
+  # U-Net
+  return unet_forward(False, cnet_info.net, original_unet, None, outs, sample, timestep, encoder_hidden_states)
+
+
+"""
+  # これはmergeのバージョン
+  # ControlNet
+  cnet_outs_list = []
+  for i, cnet_info in enumerate(control_nets):
+    # print(current_ratio, cnet_info.prep, cnet_info.weight, cnet_info.ratio)
+    if cnet_info.ratio < current_ratio:
+      continue
+    guided_hint = guided_hints[i]
+    outs = unet_forward(True, cnet_info.net, cnet_info.unet, guided_hint, None, sample, timestep, encoder_hidden_states)
+    for i in range(len(outs)):
+      outs[i] *= cnet_info.weight
+
+    cnet_outs_list.append(outs)
+
+  count = len(cnet_outs_list)
+  if count == 0:
+    return original_unet(sample, timestep, encoder_hidden_states)
+
+  # sum of controlnets
+  for i in range(1, count):
+    cnet_outs_list[0] += cnet_outs_list[i]
+
+  # U-Net
+  return unet_forward(False, cnet_info.net, original_unet, None, cnet_outs_list[0], sample, timestep, encoder_hidden_states)
+"""
+
+
+def unet_forward(is_control_net, control_net: ControlNet, unet: UNet2DConditionModel, guided_hint, ctrl_outs, sample, timestep, encoder_hidden_states):
+  # copy from UNet2DConditionModel
+  default_overall_up_factor = 2**unet.num_upsamplers
+
+  forward_upsample_size = False
+  upsample_size = None
+
+  if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):
+    print("Forward upsample size to force interpolation output size.")
+    forward_upsample_size = True
+
+  # 0. center input if necessary
+  if unet.config.center_input_sample:
+    sample = 2 * sample - 1.0
+
+  # 1. time
+  timesteps = timestep
+  if not torch.is_tensor(timesteps):
+    # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
+    # This would be a good case for the `match` statement (Python 3.10+)
+    is_mps = sample.device.type == "mps"
+    if isinstance(timestep, float):
+      dtype = torch.float32 if is_mps else torch.float64
+    else:
+      dtype = torch.int32 if is_mps else torch.int64
+    timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
+  elif len(timesteps.shape) == 0:
+    timesteps = timesteps[None].to(sample.device)
+
+  # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+  timesteps = timesteps.expand(sample.shape[0])
+
+  t_emb = unet.time_proj(timesteps)
+
+  # timesteps does not contain any weights and will always return f32 tensors
+  # but time_embedding might actually be running in fp16. so we need to cast here.
+  # there might be better ways to encapsulate this.
+  t_emb = t_emb.to(dtype=unet.dtype)
+  emb = unet.time_embedding(t_emb)
+
+  outs = []                     # output of ControlNet
+  zc_idx = 0
+
+  # 2. pre-process
+  sample = unet.conv_in(sample)
+  if is_control_net:
+    sample += guided_hint
+    outs.append(control_net.control_model.zero_convs[zc_idx][0](sample))  # , emb, encoder_hidden_states))
+    zc_idx += 1
+
+  # 3. down
+  down_block_res_samples = (sample,)
+  for downsample_block in unet.down_blocks:
+    if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention:
+      sample, res_samples = downsample_block(
+          hidden_states=sample,
+          temb=emb,
+          encoder_hidden_states=encoder_hidden_states,
+      )
+    else:
+      sample, res_samples = downsample_block(hidden_states=sample, temb=emb)
+    if is_control_net:
+      for rs in res_samples:
+        outs.append(control_net.control_model.zero_convs[zc_idx][0](rs))  # , emb, encoder_hidden_states))
+        zc_idx += 1
+
+    down_block_res_samples += res_samples
+
+  # 4. mid
+  sample = unet.mid_block(sample, emb, encoder_hidden_states=encoder_hidden_states)
+  if is_control_net:
+    outs.append(control_net.control_model.middle_block_out[0](sample))
+    return outs
+
+  if not is_control_net:
+    sample += ctrl_outs.pop()
+
+  # 5. up
+  for i, upsample_block in enumerate(unet.up_blocks):
+    is_final_block = i == len(unet.up_blocks) - 1
+
+    res_samples = down_block_res_samples[-len(upsample_block.resnets):]
+    down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)]
+
+    if not is_control_net and len(ctrl_outs) > 0:
+      res_samples = list(res_samples)
+      apply_ctrl_outs = ctrl_outs[-len(res_samples):]
+      ctrl_outs = ctrl_outs[:-len(res_samples)]
+      for j in range(len(res_samples)):
+        res_samples[j] = res_samples[j] + apply_ctrl_outs[j]
+      res_samples = tuple(res_samples)
+
+    # if we have not reached the final block and need to forward the
+    # upsample size, we do it here
+    if not is_final_block and forward_upsample_size:
+      upsample_size = down_block_res_samples[-1].shape[2:]
+
+    if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention:
+      sample = upsample_block(
+          hidden_states=sample,
+          temb=emb,
+          res_hidden_states_tuple=res_samples,
+          encoder_hidden_states=encoder_hidden_states,
+          upsample_size=upsample_size,
+      )
+    else:
+      sample = upsample_block(
+          hidden_states=sample, temb=emb, res_hidden_states_tuple=res_samples, upsample_size=upsample_size
+      )
+  # 6. post-process
+  sample = unet.conv_norm_out(sample)
+  sample = unet.conv_act(sample)
+  sample = unet.conv_out(sample)
+
+  return UNet2DConditionOutput(sample=sample)
diff --git a/tools/prepare_presets.py b/tools/prepare_presets.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2bda97b020df6193356e54546c7fd195c5789a1
--- /dev/null
+++ b/tools/prepare_presets.py
@@ -0,0 +1,32 @@
+import json
+import argparse
+import glob
+
+def remove_items_with_keywords(json_file_path):
+    keywords = ["pretrained_model_name_or_path", "dir", "save_model_as", "save_state", "resume", "output_name", "model_list", "sample_"]
+    
+    with open(json_file_path) as file:
+        data = json.load(file)
+        
+    for key in list(data.keys()):
+        for keyword in keywords:
+            if keyword in key:
+                del data[key]
+                break
+    
+    sorted_data = {k: data[k] for k in sorted(data)}
+    
+    with open(json_file_path, 'w') as file:
+        json.dump(sorted_data, file, indent=4)
+        
+    print("Items with keywords have been removed from the JSON file and the list has been sorted alphabetically:", json_file_path)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Remove items from JSON files based on keywords in the keys')
+    parser.add_argument('json_files', type=str, nargs='+', help='Path(s) to the JSON file(s)')
+    args = parser.parse_args()
+
+    json_files = args.json_files
+    for file_pattern in json_files:
+        for json_file_path in glob.glob(file_pattern):
+            remove_items_with_keywords(json_file_path)
diff --git a/tools/prune.py b/tools/prune.py
new file mode 100644
index 0000000000000000000000000000000000000000..6493bb3d5138c770ba06439bb35257638f4281fa
--- /dev/null
+++ b/tools/prune.py
@@ -0,0 +1,37 @@
+import argparse
+import torch
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description="Prune a model")
+parser.add_argument("model_prune", type=str, help="Path to model to prune")
+parser.add_argument("prune_output", type=str, help="Path to pruned ckpt output")
+parser.add_argument("--half", action="store_true", help="Save weights in half precision.")
+args = parser.parse_args()
+
+print("Loading model...")
+model_prune = torch.load(args.model_prune)
+theta_prune = model_prune["state_dict"]
+theta = {}
+
+print("Pruning model...")
+for key in tqdm(theta_prune.keys(), desc="Pruning keys"):
+    if "model" in key:
+        theta.update({key: theta_prune[key]})
+
+del theta_prune
+
+if args.half:
+    print("Halving model...")
+    state_dict = {k: v.half() for k, v in tqdm(theta.items(), desc="Halving weights")}
+else:
+    state_dict = theta
+
+del theta
+
+print("Saving pruned model...")
+
+torch.save({"state_dict": state_dict}, args.prune_output)
+
+del state_dict
+
+print("Done pruning!")
\ No newline at end of file
diff --git a/tools/rename_depth_mask.py b/tools/rename_depth_mask.py
new file mode 100644
index 0000000000000000000000000000000000000000..97efdea411bf4524e7a124a0c945c94850c4c1d4
--- /dev/null
+++ b/tools/rename_depth_mask.py
@@ -0,0 +1,21 @@
+import os
+import argparse
+
+# Define the command line arguments
+parser = argparse.ArgumentParser(description='Rename files in a folder')
+parser.add_argument('folder', metavar='folder', type=str, help='the folder containing the files to rename')
+
+# Parse the arguments
+args = parser.parse_args()
+
+# Get the list of files in the folder
+files = os.listdir(args.folder)
+
+# Loop through each file in the folder
+for file in files:
+    # Check if the file has the expected format
+    if file.endswith('-0000.png'):
+        # Get the new file name
+        new_file_name = file[:-9] + '.mask'
+        # Rename the file
+        os.rename(os.path.join(args.folder, file), os.path.join(args.folder, new_file_name))
diff --git a/tools/resize_images_to_resolution.py b/tools/resize_images_to_resolution.py
new file mode 100644
index 0000000000000000000000000000000000000000..91a496b05e00b868fb31af36280ff4e635c2f9b8
--- /dev/null
+++ b/tools/resize_images_to_resolution.py
@@ -0,0 +1,138 @@
+import glob
+import os
+import cv2
+import argparse
+import shutil
+import math
+from PIL import Image
+import numpy as np
+
+
+def resize_images(src_img_folder, dst_img_folder, max_resolution="512x512", divisible_by=2, interpolation=None, save_as_png=False, copy_associated_files=False):
+  # Split the max_resolution string by "," and strip any whitespaces
+  max_resolutions = [res.strip() for res in max_resolution.split(',')]
+
+  # # Calculate max_pixels from max_resolution string
+  # max_pixels = int(max_resolution.split("x")[0]) * int(max_resolution.split("x")[1])
+
+  # Create destination folder if it does not exist
+  if not os.path.exists(dst_img_folder):
+    os.makedirs(dst_img_folder)
+
+  # Select interpolation method
+  if interpolation == 'lanczos4':
+    cv2_interpolation = cv2.INTER_LANCZOS4
+  elif interpolation == 'cubic':
+    cv2_interpolation = cv2.INTER_CUBIC
+  else:
+    cv2_interpolation = cv2.INTER_AREA
+
+  # Iterate through all files in src_img_folder
+  img_exts = (".png", ".jpg", ".jpeg", ".webp", ".bmp")                   # copy from train_util.py
+  for filename in os.listdir(src_img_folder):
+    # Check if the image is png, jpg or webp etc...
+    if not filename.endswith(img_exts):
+      # Copy the file to the destination folder if not png, jpg or webp etc (.txt or .caption or etc.)
+      shutil.copy(os.path.join(src_img_folder, filename), os.path.join(dst_img_folder, filename))
+      continue
+
+    # Load image
+    image = Image.open(os.path.join(src_img_folder, filename))
+    if not image.mode == "RGB":
+      image = image.convert("RGB")
+    img = np.array(image, np.uint8)
+
+    base, _ = os.path.splitext(filename)
+    for max_resolution in max_resolutions:
+      # Calculate max_pixels from max_resolution string
+      max_pixels = int(max_resolution.split("x")[0]) * int(max_resolution.split("x")[1])
+
+      # Calculate current number of pixels
+      current_pixels = img.shape[0] * img.shape[1]
+
+      # Calculate current resolution
+      current_resolution = (img.shape[0], img.shape[1])
+      
+      # Calculate target resolution
+      target_resolution = (int(max_resolution.split("x")[0]), int(max_resolution.split("x")[1]))
+
+      # Skip to the next image if the current resolution is less than the target resolution
+      if current_resolution[0] < target_resolution[0] or current_resolution[1] < target_resolution[1]:
+        print(f"Skipped image: {filename} as its resolution is smaller than target resolution")
+        continue
+
+      # Check if the image needs resizing
+      if current_pixels > max_pixels:
+        # Calculate scaling factor
+        scale_factor = max_pixels / current_pixels
+
+        # Calculate new dimensions
+        new_height = int(img.shape[0] * math.sqrt(scale_factor))
+        new_width = int(img.shape[1] * math.sqrt(scale_factor))
+
+        # Resize image
+        img = cv2.resize(img, (new_width, new_height), interpolation=cv2_interpolation)
+      else:
+        new_height, new_width = img.shape[0:2]
+
+      # Calculate the new height and width that are divisible by divisible_by (with/without resizing)
+      new_height = new_height if new_height % divisible_by == 0 else new_height - new_height % divisible_by
+      new_width = new_width if new_width % divisible_by == 0 else new_width - new_width % divisible_by
+
+      # Center crop the image to the calculated dimensions
+      y = int((img.shape[0] - new_height) / 2)
+      x = int((img.shape[1] - new_width) / 2)
+      img = img[y:y + new_height, x:x + new_width]
+
+      # Split filename into base and extension
+      new_filename = base + '+' + max_resolution + ('.png' if save_as_png else '.jpg')
+
+      # Save resized image in dst_img_folder
+      # cv2.imwrite(os.path.join(dst_img_folder, new_filename), img, [cv2.IMWRITE_JPEG_QUALITY, 100])
+      image = Image.fromarray(img)
+      image.save(os.path.join(dst_img_folder, new_filename), quality=100)
+
+      proc = "Resized" if current_pixels > max_pixels else "Saved"
+      print(f"{proc} image: {filename} with size {img.shape[0]}x{img.shape[1]} as {new_filename}")
+
+    # If other files with same basename, copy them with resolution suffix
+    if copy_associated_files:
+      asoc_files = glob.glob(os.path.join(src_img_folder, base + ".*"))
+      for asoc_file in asoc_files:
+        ext = os.path.splitext(asoc_file)[1]
+        if ext in img_exts:
+          continue
+        for max_resolution in max_resolutions:
+          new_asoc_file = base + '+' + max_resolution + ext
+          print(f"Copy {asoc_file} as {new_asoc_file}")
+          shutil.copy(os.path.join(src_img_folder, asoc_file), os.path.join(dst_img_folder, new_asoc_file))
+
+
+def setup_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser(
+      description='Resize images in a folder to a specified max resolution(s) / 指定されたフォルダ内の画像を指定した最大画像サイズ（面積）以下にアスペクト比を維持したままリサイズします')
+  parser.add_argument('src_img_folder', type=str, help='Source folder containing the images / 元画像のフォルダ')
+  parser.add_argument('dst_img_folder', type=str, help='Destination folder to save the resized images / リサイズ後の画像を保存するフォルダ')
+  parser.add_argument('--max_resolution', type=str,
+                      help='Maximum resolution(s) in the format "512x512,384x384, etc, etc" / 最大画像サイズをカンマ区切りで指定 ("512x512,384x384, etc, etc" など)', default="512x512,384x384,256x256,128x128")
+  parser.add_argument('--divisible_by', type=int,
+                      help='Ensure new dimensions are divisible by this value / リサイズ後の画像のサイズをこの値で割り切れるようにします', default=1)
+  parser.add_argument('--interpolation', type=str, choices=['area', 'cubic', 'lanczos4'],
+                      default='area', help='Interpolation method for resizing / リサイズ時の補完方法')
+  parser.add_argument('--save_as_png', action='store_true', help='Save as png format / png形式で保存')
+  parser.add_argument('--copy_associated_files', action='store_true',
+                      help='Copy files with same base name to images (captions etc) / 画像と同じファイル名（拡張子を除く）のファイルもコピーする')
+
+  return parser
+
+
+def main():
+  parser = setup_parser()
+
+  args = parser.parse_args()
+  resize_images(args.src_img_folder, args.dst_img_folder, args.max_resolution,
+                args.divisible_by, args.interpolation, args.save_as_png, args.copy_associated_files)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/tools/resize_lora.py b/tools/resize_lora.py
new file mode 100644
index 0000000000000000000000000000000000000000..b99bb5bd1f485f233dea9d14889aadb3fdaa5e26
--- /dev/null
+++ b/tools/resize_lora.py
@@ -0,0 +1,339 @@
+#
+# File from: https://raw.githubusercontent.com/mgz-dev/sd-scripts/main/networks/resize_lora.py
+#
+
+# Convert LoRA to different rank approximation (should only be used to go to lower rank)
+# This code is based off the extract_lora_from_models.py file which is based on https://github.com/cloneofsimo/lora/blob/develop/lora_diffusion/cli_svd.py
+# Thanks to cloneofsimo and kohya
+
+import argparse
+import torch
+from safetensors.torch import load_file, save_file, safe_open
+from tqdm import tqdm
+from library import train_util, model_util
+import numpy as np
+
+MIN_SV = 1e-6
+
+def load_state_dict(file_name, dtype):
+  if model_util.is_safetensors(file_name):
+    sd = load_file(file_name)
+    with safe_open(file_name, framework="pt") as f:
+      metadata = f.metadata()
+  else:
+    sd = torch.load(file_name, map_location='cpu')
+    metadata = None
+
+  for key in list(sd.keys()):
+    if type(sd[key]) == torch.Tensor:
+      sd[key] = sd[key].to(dtype)
+
+  return sd, metadata
+
+
+def save_to_file(file_name, model, state_dict, dtype, metadata):
+  if dtype is not None:
+    for key in list(state_dict.keys()):
+      if type(state_dict[key]) == torch.Tensor:
+        state_dict[key] = state_dict[key].to(dtype)
+
+  if model_util.is_safetensors(file_name):
+    save_file(model, file_name, metadata)
+  else:
+    torch.save(model, file_name)
+
+
+def index_sv_cumulative(S, target):
+  original_sum = float(torch.sum(S))
+  cumulative_sums = torch.cumsum(S, dim=0)/original_sum
+  index = int(torch.searchsorted(cumulative_sums, target)) + 1
+  if index >= len(S):
+    index = len(S) - 1
+
+  return index
+
+
+def index_sv_fro(S, target):
+  S_squared = S.pow(2)
+  s_fro_sq = float(torch.sum(S_squared))
+  sum_S_squared = torch.cumsum(S_squared, dim=0)/s_fro_sq
+  index = int(torch.searchsorted(sum_S_squared, target**2)) + 1
+  if index >= len(S):
+    index = len(S) - 1
+
+  return index
+
+
+# Modified from Kohaku-blueleaf's extract/merge functions
+def extract_conv(weight, lora_rank, dynamic_method, dynamic_param, device, scale=1):
+    out_size, in_size, kernel_size, _ = weight.size()
+    U, S, Vh = torch.linalg.svd(weight.reshape(out_size, -1).to(device))
+    
+    param_dict = rank_resize(S, lora_rank, dynamic_method, dynamic_param, scale)
+    lora_rank = param_dict["new_rank"]
+
+    U = U[:, :lora_rank]
+    S = S[:lora_rank]
+    U = U @ torch.diag(S)
+    Vh = Vh[:lora_rank, :]
+
+    param_dict["lora_down"] = Vh.reshape(lora_rank, in_size, kernel_size, kernel_size).cpu()
+    param_dict["lora_up"] = U.reshape(out_size, lora_rank, 1, 1).cpu()
+    del U, S, Vh, weight
+    return param_dict
+
+
+def extract_linear(weight, lora_rank, dynamic_method, dynamic_param, device, scale=1):
+    out_size, in_size = weight.size()
+    
+    U, S, Vh = torch.linalg.svd(weight.to(device))
+    
+    param_dict = rank_resize(S, lora_rank, dynamic_method, dynamic_param, scale)
+    lora_rank = param_dict["new_rank"]
+    
+    U = U[:, :lora_rank]
+    S = S[:lora_rank]
+    U = U @ torch.diag(S)
+    Vh = Vh[:lora_rank, :]
+    
+    param_dict["lora_down"] = Vh.reshape(lora_rank, in_size).cpu()
+    param_dict["lora_up"] = U.reshape(out_size, lora_rank).cpu()
+    del U, S, Vh, weight
+    return param_dict
+
+
+def merge_conv(lora_down, lora_up, device):
+    in_rank, in_size, kernel_size, k_ = lora_down.shape
+    out_size, out_rank, _, _ = lora_up.shape
+    assert in_rank == out_rank and kernel_size == k_, f"rank {in_rank} {out_rank} or kernel {kernel_size} {k_} mismatch"
+    
+    lora_down = lora_down.to(device)
+    lora_up = lora_up.to(device)
+
+    merged = lora_up.reshape(out_size, -1) @ lora_down.reshape(in_rank, -1)
+    weight = merged.reshape(out_size, in_size, kernel_size, kernel_size)
+    del lora_up, lora_down
+    return weight
+
+
+def merge_linear(lora_down, lora_up, device):
+    in_rank, in_size = lora_down.shape
+    out_size, out_rank = lora_up.shape
+    assert in_rank == out_rank, f"rank {in_rank} {out_rank} mismatch"
+    
+    lora_down = lora_down.to(device)
+    lora_up = lora_up.to(device)
+    
+    weight = lora_up @ lora_down
+    del lora_up, lora_down
+    return weight
+  
+
+def rank_resize(S, rank, dynamic_method, dynamic_param, scale=1):
+    param_dict = {}
+
+    if dynamic_method=="sv_ratio":
+        # Calculate new dim and alpha based off ratio
+        max_sv = S[0]
+        min_sv = max_sv/dynamic_param
+        new_rank = max(torch.sum(S > min_sv).item(),1)
+        new_alpha = float(scale*new_rank)
+
+    elif dynamic_method=="sv_cumulative":
+        # Calculate new dim and alpha based off cumulative sum
+        new_rank = index_sv_cumulative(S, dynamic_param)
+        new_rank = max(new_rank, 1)
+        new_alpha = float(scale*new_rank)
+
+    elif dynamic_method=="sv_fro":
+        # Calculate new dim and alpha based off sqrt sum of squares
+        new_rank = index_sv_fro(S, dynamic_param)
+        new_rank = min(max(new_rank, 1), len(S)-1)
+        new_alpha = float(scale*new_rank)
+    else:
+        new_rank = rank
+        new_alpha = float(scale*new_rank)
+
+    
+    if S[0] <= MIN_SV: # Zero matrix, set dim to 1
+        new_rank = 1
+        new_alpha = float(scale*new_rank)
+    elif new_rank > rank: # cap max rank at rank
+        new_rank = rank
+        new_alpha = float(scale*new_rank)
+
+
+    # Calculate resize info
+    s_sum = torch.sum(torch.abs(S))
+    s_rank = torch.sum(torch.abs(S[:new_rank]))
+    
+    S_squared = S.pow(2)
+    s_fro = torch.sqrt(torch.sum(S_squared))
+    s_red_fro = torch.sqrt(torch.sum(S_squared[:new_rank]))
+    fro_percent = float(s_red_fro/s_fro)
+
+    param_dict["new_rank"] = new_rank
+    param_dict["new_alpha"] = new_alpha
+    param_dict["sum_retained"] = (s_rank)/s_sum
+    param_dict["fro_retained"] = fro_percent
+    param_dict["max_ratio"] = S[0]/S[new_rank]
+
+    return param_dict
+
+
+def resize_lora_model(lora_sd, new_rank, save_dtype, device, dynamic_method, dynamic_param, verbose):
+  network_alpha = None
+  network_dim = None
+  verbose_str = "\n"
+  fro_list = []
+
+  # Extract loaded lora dim and alpha
+  for key, value in lora_sd.items():
+    if network_alpha is None and 'alpha' in key:
+      network_alpha = value
+    if network_dim is None and 'lora_down' in key and len(value.size()) == 2:
+      network_dim = value.size()[0]
+    if network_alpha is not None and network_dim is not None:
+      break
+    if network_alpha is None:
+      network_alpha = network_dim
+
+  scale = network_alpha/network_dim
+
+  if dynamic_method:
+    print(f"Dynamically determining new alphas and dims based off {dynamic_method}: {dynamic_param}, max rank is {new_rank}")
+
+  lora_down_weight = None
+  lora_up_weight = None
+
+  o_lora_sd = lora_sd.copy()
+  block_down_name = None
+  block_up_name = None
+
+  with torch.no_grad():
+    for key, value in tqdm(lora_sd.items()):
+      if 'lora_down' in key:
+        block_down_name = key.split(".")[0]
+        lora_down_weight = value
+      if 'lora_up' in key:
+        block_up_name = key.split(".")[0]
+        lora_up_weight = value
+
+      weights_loaded = (lora_down_weight is not None and lora_up_weight is not None)
+
+      if (block_down_name == block_up_name) and weights_loaded:
+
+        conv2d = (len(lora_down_weight.size()) == 4)
+
+        if conv2d:
+          full_weight_matrix = merge_conv(lora_down_weight, lora_up_weight, device)
+          param_dict = extract_conv(full_weight_matrix, new_rank, dynamic_method, dynamic_param, device, scale)
+        else:
+          full_weight_matrix = merge_linear(lora_down_weight, lora_up_weight, device)
+          param_dict = extract_linear(full_weight_matrix, new_rank, dynamic_method, dynamic_param, device, scale)
+
+        if verbose:
+          max_ratio = param_dict['max_ratio']
+          sum_retained = param_dict['sum_retained']
+          fro_retained = param_dict['fro_retained']
+          if not np.isnan(fro_retained):
+            fro_list.append(float(fro_retained))
+
+          verbose_str+=f"{block_down_name:75} | "
+          verbose_str+=f"sum(S) retained: {sum_retained:.1%}, fro retained: {fro_retained:.1%}, max(S) ratio: {max_ratio:0.1f}"
+
+        if verbose and dynamic_method:
+          verbose_str+=f", dynamic | dim: {param_dict['new_rank']}, alpha: {param_dict['new_alpha']}\n"
+        else:
+          verbose_str+=f"\n"
+
+        new_alpha = param_dict['new_alpha']
+        o_lora_sd[block_down_name + "." + "lora_down.weight"] = param_dict["lora_down"].to(save_dtype).contiguous()
+        o_lora_sd[block_up_name + "." + "lora_up.weight"] = param_dict["lora_up"].to(save_dtype).contiguous()
+        o_lora_sd[block_up_name + "." "alpha"] = torch.tensor(param_dict['new_alpha']).to(save_dtype)
+
+        block_down_name = None
+        block_up_name = None
+        lora_down_weight = None
+        lora_up_weight = None
+        weights_loaded = False
+        del param_dict
+
+  if verbose:
+    print(verbose_str)
+
+    print(f"Average Frobenius norm retention: {np.mean(fro_list):.2%} | std: {np.std(fro_list):0.3f}")
+  print("resizing complete")
+  return o_lora_sd, network_dim, new_alpha
+
+
+def resize(args):
+
+  def str_to_dtype(p):
+    if p == 'float':
+      return torch.float
+    if p == 'fp16':
+      return torch.float16
+    if p == 'bf16':
+      return torch.bfloat16
+    return None
+
+  if args.dynamic_method and not args.dynamic_param:
+    raise Exception("If using dynamic_method, then dynamic_param is required")
+
+  merge_dtype = str_to_dtype('float')  # matmul method above only seems to work in float32
+  save_dtype = str_to_dtype(args.save_precision)
+  if save_dtype is None:
+    save_dtype = merge_dtype
+
+  print("loading Model...")
+  lora_sd, metadata = load_state_dict(args.model, merge_dtype)
+
+  print("Resizing Lora...")
+  state_dict, old_dim, new_alpha = resize_lora_model(lora_sd, args.new_rank, save_dtype, args.device, args.dynamic_method, args.dynamic_param, args.verbose)
+
+  # update metadata
+  if metadata is None:
+    metadata = {}
+
+  comment = metadata.get("ss_training_comment", "")
+
+  if not args.dynamic_method:
+    metadata["ss_training_comment"] = f"dimension is resized from {old_dim} to {args.new_rank}; {comment}"
+    metadata["ss_network_dim"] = str(args.new_rank)
+    metadata["ss_network_alpha"] = str(new_alpha)
+  else:
+    metadata["ss_training_comment"] = f"Dynamic resize with {args.dynamic_method}: {args.dynamic_param} from {old_dim}; {comment}"
+    metadata["ss_network_dim"] = 'Dynamic'
+    metadata["ss_network_alpha"] = 'Dynamic'
+
+  model_hash, legacy_hash = train_util.precalculate_safetensors_hashes(state_dict, metadata)
+  metadata["sshs_model_hash"] = model_hash
+  metadata["sshs_legacy_hash"] = legacy_hash
+
+  print(f"saving model to: {args.save_to}")
+  save_to_file(args.save_to, state_dict, state_dict, save_dtype, metadata)
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument("--save_precision", type=str, default=None,
+                      choices=[None, "float", "fp16", "bf16"], help="precision in saving, float if omitted / 保存時の精度、未指定時はfloat")
+  parser.add_argument("--new_rank", type=int, default=4,
+                      help="Specify rank of output LoRA / 出力するLoRAのrank (dim)")
+  parser.add_argument("--save_to", type=str, default=None,
+                      help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors")
+  parser.add_argument("--model", type=str, default=None,
+                      help="LoRA model to resize at to new rank: ckpt or safetensors file / 読み込むLoRAモデル、ckptまたはsafetensors")
+  parser.add_argument("--device", type=str, default=None, help="device to use, cuda for GPU / 計算を行うデバイス、cuda でGPUを使う")
+  parser.add_argument("--verbose", action="store_true", 
+                      help="Display verbose resizing information / rank変更時の詳細情報を出力する")
+  parser.add_argument("--dynamic_method", type=str, default=None, choices=[None, "sv_ratio", "sv_fro", "sv_cumulative"],
+                      help="Specify dynamic resizing method, --new_rank is used as a hard limit for max rank")
+  parser.add_argument("--dynamic_param", type=float, default=None,
+                      help="Specify target for dynamic reduction")
+                                           
+
+  args = parser.parse_args()
+  resize(args)
\ No newline at end of file
diff --git a/train_controlnet.py b/train_controlnet.py
new file mode 100644
index 0000000000000000000000000000000000000000..39ac43e9671cbd7d0015de8ff9523255ad76a0f3
--- /dev/null
+++ b/train_controlnet.py
@@ -0,0 +1,600 @@
+import argparse
+import gc
+import json
+import math
+import os
+import random
+import time
+from multiprocessing import Value
+from types import SimpleNamespace
+
+from tqdm import tqdm
+import torch
+from torch.nn.parallel import DistributedDataParallel as DDP
+from accelerate.utils import set_seed
+from diffusers import DDPMScheduler, ControlNetModel
+from safetensors.torch import load_file
+
+import library.model_util as model_util
+import library.train_util as train_util
+import library.config_util as config_util
+from library.config_util import (
+    ConfigSanitizer,
+    BlueprintGenerator,
+)
+import library.huggingface_util as huggingface_util
+import library.custom_train_functions as custom_train_functions
+from library.custom_train_functions import (
+    apply_snr_weight,
+    pyramid_noise_like,
+    apply_noise_offset,
+)
+
+
+# TODO 他のスクリプトと共通化する
+def generate_step_logs(args: argparse.Namespace, current_loss, avr_loss, lr_scheduler):
+    logs = {
+        "loss/current": current_loss,
+        "loss/average": avr_loss,
+        "lr": lr_scheduler.get_last_lr()[0],
+    }
+
+    if args.optimizer_type.lower().startswith("DAdapt".lower()):
+        logs["lr/d*lr"] = lr_scheduler.optimizers[-1].param_groups[0]["d"] * lr_scheduler.optimizers[-1].param_groups[0]["lr"]
+
+    return logs
+
+
+def train(args):
+    # session_id = random.randint(0, 2**32)
+    # training_started_at = time.time()
+    train_util.verify_training_args(args)
+    train_util.prepare_dataset_args(args, True)
+
+    cache_latents = args.cache_latents
+    use_user_config = args.dataset_config is not None
+
+    if args.seed is None:
+        args.seed = random.randint(0, 2**32)
+    set_seed(args.seed)
+
+    tokenizer = train_util.load_tokenizer(args)
+
+    # データセットを準備する
+    blueprint_generator = BlueprintGenerator(ConfigSanitizer(False, False, True, True))
+    if use_user_config:
+        print(f"Load dataset config from {args.dataset_config}")
+        user_config = config_util.load_user_config(args.dataset_config)
+        ignored = ["train_data_dir", "conditioning_data_dir"]
+        if any(getattr(args, attr) is not None for attr in ignored):
+            print(
+                "ignore following options because config file is found: {0} / 設定ファイルが利用されるため以下のオプションは無視されます: {0}".format(
+                    ", ".join(ignored)
+                )
+            )
+    else:
+        user_config = {
+            "datasets": [
+                {
+                    "subsets": config_util.generate_controlnet_subsets_config_by_subdirs(
+                        args.train_data_dir,
+                        args.conditioning_data_dir,
+                        args.caption_extension,
+                    )
+                }
+            ]
+        }
+
+    blueprint = blueprint_generator.generate(user_config, args, tokenizer=tokenizer)
+    train_dataset_group = config_util.generate_dataset_group_by_blueprint(blueprint.dataset_group)
+
+    current_epoch = Value("i", 0)
+    current_step = Value("i", 0)
+    ds_for_collater = train_dataset_group if args.max_data_loader_n_workers == 0 else None
+    collater = train_util.collater_class(current_epoch, current_step, ds_for_collater)
+
+    if args.debug_dataset:
+        train_util.debug_dataset(train_dataset_group)
+        return
+    if len(train_dataset_group) == 0:
+        print(
+            "No data found. Please verify arguments (train_data_dir must be the parent of folders with images) / 画像がありません。引数指定を確認してください（train_data_dirには画像があるフォルダではなく、画像があるフォルダの親フォルダを指定する必要があります）"
+        )
+        return
+
+    if cache_latents:
+        assert (
+            train_dataset_group.is_latent_cacheable()
+        ), "when caching latents, either color_aug or random_crop cannot be used / latentをキャッシュするときはcolor_augとrandom_cropは使えません"
+
+    # acceleratorを準備する
+    print("prepare accelerator")
+    accelerator = train_util.prepare_accelerator(args)
+    is_main_process = accelerator.is_main_process
+
+    # mixed precisionに対応した型を用意しておき適宜castする
+    weight_dtype, save_dtype = train_util.prepare_dtype(args)
+
+    # モデルを読み込む
+    text_encoder, vae, unet, _ = train_util.load_target_model(
+        args, weight_dtype, accelerator, unet_use_linear_projection_in_v2=True
+    )
+
+    # DiffusersのControlNetが使用するデータを準備する
+    if args.v2:
+        unet.config = {
+            "act_fn": "silu",
+            "attention_head_dim": [5, 10, 20, 20],
+            "block_out_channels": [320, 640, 1280, 1280],
+            "center_input_sample": False,
+            "cross_attention_dim": 1024,
+            "down_block_types": ["CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D"],
+            "downsample_padding": 1,
+            "dual_cross_attention": False,
+            "flip_sin_to_cos": True,
+            "freq_shift": 0,
+            "in_channels": 4,
+            "layers_per_block": 2,
+            "mid_block_scale_factor": 1,
+            "norm_eps": 1e-05,
+            "norm_num_groups": 32,
+            "num_class_embeds": None,
+            "only_cross_attention": False,
+            "out_channels": 4,
+            "sample_size": 96,
+            "up_block_types": ["UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"],
+            "use_linear_projection": True,
+            "upcast_attention": True,
+            "only_cross_attention": False,
+            "downsample_padding": 1,
+            "use_linear_projection": True,
+            "class_embed_type": None,
+            "num_class_embeds": None,
+            "resnet_time_scale_shift": "default",
+            "projection_class_embeddings_input_dim": None,
+        }
+    else:
+        unet.config = {
+            "act_fn": "silu",
+            "attention_head_dim": 8,
+            "block_out_channels": [320, 640, 1280, 1280],
+            "center_input_sample": False,
+            "cross_attention_dim": 768,
+            "down_block_types": ["CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D"],
+            "downsample_padding": 1,
+            "flip_sin_to_cos": True,
+            "freq_shift": 0,
+            "in_channels": 4,
+            "layers_per_block": 2,
+            "mid_block_scale_factor": 1,
+            "norm_eps": 1e-05,
+            "norm_num_groups": 32,
+            "out_channels": 4,
+            "sample_size": 64,
+            "up_block_types": ["UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"],
+            "only_cross_attention": False,
+            "downsample_padding": 1,
+            "use_linear_projection": False,
+            "class_embed_type": None,
+            "num_class_embeds": None,
+            "upcast_attention": False,
+            "resnet_time_scale_shift": "default",
+            "projection_class_embeddings_input_dim": None,
+        }
+    unet.config = SimpleNamespace(**unet.config)
+
+    controlnet = ControlNetModel.from_unet(unet)
+
+    if args.controlnet_model_name_or_path:
+        filename = args.controlnet_model_name_or_path
+        if os.path.isfile(filename):
+            if os.path.splitext(filename)[1] == ".safetensors":
+                state_dict = load_file(filename)
+            else:
+                state_dict = torch.load(filename)
+            state_dict = model_util.convert_controlnet_state_dict_to_diffusers(state_dict)
+            controlnet.load_state_dict(state_dict)
+        elif os.path.isdir(filename):
+            controlnet = ControlNetModel.from_pretrained(filename)
+
+    # モデルに xformers とか memory efficient attention を組み込む
+    train_util.replace_unet_modules(unet, args.mem_eff_attn, args.xformers, args.sdpa)
+
+    # 学習を準備する
+    if cache_latents:
+        vae.to(accelerator.device, dtype=weight_dtype)
+        vae.requires_grad_(False)
+        vae.eval()
+        with torch.no_grad():
+            train_dataset_group.cache_latents(
+                vae,
+                args.vae_batch_size,
+                args.cache_latents_to_disk,
+                accelerator.is_main_process,
+            )
+        vae.to("cpu")
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        gc.collect()
+
+        accelerator.wait_for_everyone()
+
+    if args.gradient_checkpointing:
+        controlnet.enable_gradient_checkpointing()
+
+    # 学習に必要なクラスを準備する
+    accelerator.print("prepare optimizer, data loader etc.")
+
+    trainable_params = controlnet.parameters()
+
+    _, _, optimizer = train_util.get_optimizer(args, trainable_params)
+
+    # dataloaderを準備する
+    # DataLoaderのプロセス数：0はメインプロセスになる
+    n_workers = min(args.max_data_loader_n_workers, os.cpu_count() - 1)  # cpu_count-1 ただし最大で指定された数まで
+
+    train_dataloader = torch.utils.data.DataLoader(
+        train_dataset_group,
+        batch_size=1,
+        shuffle=True,
+        collate_fn=collater,
+        num_workers=n_workers,
+        persistent_workers=args.persistent_data_loader_workers,
+    )
+
+    # 学習ステップ数を計算する
+    if args.max_train_epochs is not None:
+        args.max_train_steps = args.max_train_epochs * math.ceil(
+            len(train_dataloader) / accelerator.num_processes / args.gradient_accumulation_steps
+        )
+        accelerator.print(f"override steps. steps for {args.max_train_epochs} epochs is / 指定エポックまでのステップ数: {args.max_train_steps}")
+
+    # データセット側にも学習ステップを送信
+    train_dataset_group.set_max_train_steps(args.max_train_steps)
+
+    # lr schedulerを用意する
+    lr_scheduler = train_util.get_scheduler_fix(args, optimizer, accelerator.num_processes)
+
+    # 実験的機能：勾配も含めたfp16学習を行う　モデル全体をfp16にする
+    if args.full_fp16:
+        assert (
+            args.mixed_precision == "fp16"
+        ), "full_fp16 requires mixed precision='fp16' / full_fp16を使う場合はmixed_precision='fp16'を指定してください。"
+        accelerator.print("enable full fp16 training.")
+        controlnet.to(weight_dtype)
+
+    # acceleratorがなんかよろしくやってくれるらしい
+    controlnet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+        controlnet, optimizer, train_dataloader, lr_scheduler
+    )
+
+    unet.requires_grad_(False)
+    text_encoder.requires_grad_(False)
+    unet.to(accelerator.device)
+    text_encoder.to(accelerator.device)
+
+    # transform DDP after prepare
+    controlnet = controlnet.module if isinstance(controlnet, DDP) else controlnet
+
+    controlnet.train()
+
+    if not cache_latents:
+        vae.requires_grad_(False)
+        vae.eval()
+        vae.to(accelerator.device, dtype=weight_dtype)
+
+    # 実験的機能：勾配も含めたfp16学習を行う　PyTorchにパッチを当ててfp16でのgrad scaleを有効にする
+    if args.full_fp16:
+        train_util.patch_accelerator_for_fp16_training(accelerator)
+
+    # resumeする
+    train_util.resume_from_local_or_hf_if_specified(accelerator, args)
+
+    # epoch数を計算する
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+    if (args.save_n_epoch_ratio is not None) and (args.save_n_epoch_ratio > 0):
+        args.save_every_n_epochs = math.floor(num_train_epochs / args.save_n_epoch_ratio) or 1
+
+    # 学習する
+    # TODO: find a way to handle total batch size when there are multiple datasets
+    accelerator.print("running training / 学習開始")
+    accelerator.print(f"  num train images * repeats / 学習画像の数×繰り返し回数: {train_dataset_group.num_train_images}")
+    accelerator.print(f"  num reg images / 正則化画像の数: {train_dataset_group.num_reg_images}")
+    accelerator.print(f"  num batches per epoch / 1epochのバッチ数: {len(train_dataloader)}")
+    accelerator.print(f"  num epochs / epoch数: {num_train_epochs}")
+    accelerator.print(f"  batch size per device / バッチサイズ: {', '.join([str(d.batch_size) for d in train_dataset_group.datasets])}")
+    # print(f"  total train batch size (with parallel & distributed & accumulation) / 総バッチサイズ（並列学習、勾配合計含む）: {total_batch_size}")
+    accelerator.print(f"  gradient accumulation steps / 勾配を合計するステップ数 = {args.gradient_accumulation_steps}")
+    accelerator.print(f"  total optimization steps / 学習ステップ数: {args.max_train_steps}")
+
+    progress_bar = tqdm(
+        range(args.max_train_steps),
+        smoothing=0,
+        disable=not accelerator.is_local_main_process,
+        desc="steps",
+    )
+    global_step = 0
+
+    noise_scheduler = DDPMScheduler(
+        beta_start=0.00085,
+        beta_end=0.012,
+        beta_schedule="scaled_linear",
+        num_train_timesteps=1000,
+        clip_sample=False,
+    )
+    if accelerator.is_main_process:
+        accelerator.init_trackers("controlnet_train" if args.log_tracker_name is None else args.log_tracker_name)
+
+    loss_list = []
+    loss_total = 0.0
+    del train_dataset_group
+
+    # function for saving/removing
+    def save_model(ckpt_name, model, force_sync_upload=False):
+        os.makedirs(args.output_dir, exist_ok=True)
+        ckpt_file = os.path.join(args.output_dir, ckpt_name)
+
+        accelerator.print(f"\nsaving checkpoint: {ckpt_file}")
+
+        state_dict = model_util.convert_controlnet_state_dict_to_sd(model.state_dict())
+
+        if save_dtype is not None:
+            for key in list(state_dict.keys()):
+                v = state_dict[key]
+                v = v.detach().clone().to("cpu").to(save_dtype)
+                state_dict[key] = v
+
+        if os.path.splitext(ckpt_file)[1] == ".safetensors":
+            from safetensors.torch import save_file
+
+            save_file(state_dict, ckpt_file)
+        else:
+            torch.save(state_dict, ckpt_file)
+
+        if args.huggingface_repo_id is not None:
+            huggingface_util.upload(args, ckpt_file, "/" + ckpt_name, force_sync_upload=force_sync_upload)
+
+    def remove_model(old_ckpt_name):
+        old_ckpt_file = os.path.join(args.output_dir, old_ckpt_name)
+        if os.path.exists(old_ckpt_file):
+            accelerator.print(f"removing old checkpoint: {old_ckpt_file}")
+            os.remove(old_ckpt_file)
+
+    # training loop
+    for epoch in range(num_train_epochs):
+        if is_main_process:
+            accelerator.print(f"\nepoch {epoch+1}/{num_train_epochs}")
+        current_epoch.value = epoch + 1
+
+        for step, batch in enumerate(train_dataloader):
+            current_step.value = global_step
+            with accelerator.accumulate(controlnet):
+                with torch.no_grad():
+                    if "latents" in batch and batch["latents"] is not None:
+                        latents = batch["latents"].to(accelerator.device)
+                    else:
+                        # latentに変換
+                        latents = vae.encode(batch["images"].to(dtype=weight_dtype)).latent_dist.sample()
+                    latents = latents * 0.18215
+                b_size = latents.shape[0]
+
+                input_ids = batch["input_ids"].to(accelerator.device)
+                encoder_hidden_states = train_util.get_hidden_states(args, input_ids, tokenizer, text_encoder, weight_dtype)
+
+                # Sample noise that we'll add to the latents
+                noise = torch.randn_like(latents, device=latents.device)
+                if args.noise_offset:
+                    noise = apply_noise_offset(latents, noise, args.noise_offset, args.adaptive_noise_scale)
+                elif args.multires_noise_iterations:
+                    noise = pyramid_noise_like(
+                        noise,
+                        latents.device,
+                        args.multires_noise_iterations,
+                        args.multires_noise_discount,
+                    )
+
+                # Sample a random timestep for each image
+                timesteps = torch.randint(
+                    0,
+                    noise_scheduler.config.num_train_timesteps,
+                    (b_size,),
+                    device=latents.device,
+                )
+                timesteps = timesteps.long()
+                # Add noise to the latents according to the noise magnitude at each timestep
+                # (this is the forward diffusion process)
+                noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps)
+
+                controlnet_image = batch["conditioning_images"].to(dtype=weight_dtype)
+
+                with accelerator.autocast():
+                    down_block_res_samples, mid_block_res_sample = controlnet(
+                        noisy_latents,
+                        timesteps,
+                        encoder_hidden_states=encoder_hidden_states,
+                        controlnet_cond=controlnet_image,
+                        return_dict=False,
+                    )
+
+                    # Predict the noise residual
+                    noise_pred = unet(
+                        noisy_latents,
+                        timesteps,
+                        encoder_hidden_states,
+                        down_block_additional_residuals=[sample.to(dtype=weight_dtype) for sample in down_block_res_samples],
+                        mid_block_additional_residual=mid_block_res_sample.to(dtype=weight_dtype),
+                    ).sample
+
+                if args.v_parameterization:
+                    # v-parameterization training
+                    target = noise_scheduler.get_velocity(latents, noise, timesteps)
+                else:
+                    target = noise
+
+                loss = torch.nn.functional.mse_loss(noise_pred.float(), target.float(), reduction="none")
+                loss = loss.mean([1, 2, 3])
+
+                loss_weights = batch["loss_weights"]  # 各sampleごとのweight
+                loss = loss * loss_weights
+
+                if args.min_snr_gamma:
+                    loss = apply_snr_weight(loss, timesteps, noise_scheduler, args.min_snr_gamma)
+
+                loss = loss.mean()  # 平均なのでbatch_sizeで割る必要なし
+
+                accelerator.backward(loss)
+                if accelerator.sync_gradients and args.max_grad_norm != 0.0:
+                    params_to_clip = controlnet.parameters()
+                    accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
+
+                optimizer.step()
+                lr_scheduler.step()
+                optimizer.zero_grad(set_to_none=True)
+
+            # Checks if the accelerator has performed an optimization step behind the scenes
+            if accelerator.sync_gradients:
+                progress_bar.update(1)
+                global_step += 1
+
+                train_util.sample_images(
+                    accelerator,
+                    args,
+                    None,
+                    global_step,
+                    accelerator.device,
+                    vae,
+                    tokenizer,
+                    text_encoder,
+                    unet,
+                    controlnet=controlnet,
+                )
+
+                # 指定ステップごとにモデルを保存
+                if args.save_every_n_steps is not None and global_step % args.save_every_n_steps == 0:
+                    accelerator.wait_for_everyone()
+                    if accelerator.is_main_process:
+                        ckpt_name = train_util.get_step_ckpt_name(args, "." + args.save_model_as, global_step)
+                        save_model(
+                            ckpt_name,
+                            accelerator.unwrap_model(controlnet),
+                        )
+
+                        if args.save_state:
+                            train_util.save_and_remove_state_stepwise(args, accelerator, global_step)
+
+                        remove_step_no = train_util.get_remove_step_no(args, global_step)
+                        if remove_step_no is not None:
+                            remove_ckpt_name = train_util.get_step_ckpt_name(args, "." + args.save_model_as, remove_step_no)
+                            remove_model(remove_ckpt_name)
+
+            current_loss = loss.detach().item()
+            if epoch == 0:
+                loss_list.append(current_loss)
+            else:
+                loss_total -= loss_list[step]
+                loss_list[step] = current_loss
+            loss_total += current_loss
+            avr_loss = loss_total / len(loss_list)
+            logs = {"loss": avr_loss}  # , "lr": lr_scheduler.get_last_lr()[0]}
+            progress_bar.set_postfix(**logs)
+
+            if args.logging_dir is not None:
+                logs = generate_step_logs(args, current_loss, avr_loss, lr_scheduler)
+                accelerator.log(logs, step=global_step)
+
+            if global_step >= args.max_train_steps:
+                break
+
+        if args.logging_dir is not None:
+            logs = {"loss/epoch": loss_total / len(loss_list)}
+            accelerator.log(logs, step=epoch + 1)
+
+        accelerator.wait_for_everyone()
+
+        # 指定エポックごとにモデルを保存
+        if args.save_every_n_epochs is not None:
+            saving = (epoch + 1) % args.save_every_n_epochs == 0 and (epoch + 1) < num_train_epochs
+            if is_main_process and saving:
+                ckpt_name = train_util.get_epoch_ckpt_name(args, "." + args.save_model_as, epoch + 1)
+                save_model(ckpt_name, accelerator.unwrap_model(controlnet))
+
+                remove_epoch_no = train_util.get_remove_epoch_no(args, epoch + 1)
+                if remove_epoch_no is not None:
+                    remove_ckpt_name = train_util.get_epoch_ckpt_name(args, "." + args.save_model_as, remove_epoch_no)
+                    remove_model(remove_ckpt_name)
+
+                if args.save_state:
+                    train_util.save_and_remove_state_on_epoch_end(args, accelerator, epoch + 1)
+
+        train_util.sample_images(
+            accelerator,
+            args,
+            epoch + 1,
+            global_step,
+            accelerator.device,
+            vae,
+            tokenizer,
+            text_encoder,
+            unet,
+            controlnet=controlnet,
+        )
+
+        # end of epoch
+    if is_main_process:
+        controlnet = accelerator.unwrap_model(controlnet)
+
+    accelerator.end_training()
+
+    if is_main_process and args.save_state:
+        train_util.save_state_on_train_end(args, accelerator)
+
+    # del accelerator  # この後メモリを使うのでこれは消す→printで使うので消さずにおく
+
+    if is_main_process:
+        ckpt_name = train_util.get_last_ckpt_name(args, "." + args.save_model_as)
+        save_model(ckpt_name, controlnet, force_sync_upload=True)
+
+        print("model saved.")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    train_util.add_sd_models_arguments(parser)
+    train_util.add_dataset_arguments(parser, False, True, True)
+    train_util.add_training_arguments(parser, False)
+    train_util.add_optimizer_arguments(parser)
+    config_util.add_config_arguments(parser)
+    custom_train_functions.add_custom_train_arguments(parser)
+
+    parser.add_argument(
+        "--save_model_as",
+        type=str,
+        default="safetensors",
+        choices=[None, "ckpt", "pt", "safetensors"],
+        help="format to save the model (default is .safetensors) / モデル保存時の形式（デフォルトはsafetensors）",
+    )
+    parser.add_argument(
+        "--controlnet_model_name_or_path",
+        type=str,
+        default=None,
+        help="controlnet model name or path / controlnetのモデル名またはパス",
+    )
+    parser.add_argument(
+        "--conditioning_data_dir",
+        type=str,
+        default=None,
+        help="conditioning data directory / 条件付けデータのディレクトリ",
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    args = train_util.read_config_from_file(args, parser)
+
+    train(args)
diff --git a/train_db.py b/train_db.py
new file mode 100644
index 0000000000000000000000000000000000000000..439f4b9d9274ebdad24d7948629fad61ff12ad3a
--- /dev/null
+++ b/train_db.py
@@ -0,0 +1,474 @@
+# DreamBooth training
+# XXX dropped option: fine_tune
+
+import gc
+import argparse
+import itertools
+import math
+import os
+from multiprocessing import Value
+
+from tqdm import tqdm
+import torch
+from accelerate.utils import set_seed
+from diffusers import DDPMScheduler
+
+import library.train_util as train_util
+import library.config_util as config_util
+from library.config_util import (
+    ConfigSanitizer,
+    BlueprintGenerator,
+)
+import library.custom_train_functions as custom_train_functions
+from library.custom_train_functions import (
+    apply_snr_weight,
+    get_weighted_text_embeddings,
+    prepare_scheduler_for_custom_training,
+    pyramid_noise_like,
+    apply_noise_offset,
+    scale_v_prediction_loss_like_noise_prediction,
+)
+
+# perlin_noise,
+
+
+def train(args):
+    train_util.verify_training_args(args)
+    train_util.prepare_dataset_args(args, False)
+
+    cache_latents = args.cache_latents
+
+    if args.seed is not None:
+        set_seed(args.seed)  # 乱数系列を初期化する
+
+    tokenizer = train_util.load_tokenizer(args)
+
+    # データセットを準備する
+    if args.dataset_class is None:
+        blueprint_generator = BlueprintGenerator(ConfigSanitizer(True, False, False, True))
+        if args.dataset_config is not None:
+            print(f"Load dataset config from {args.dataset_config}")
+            user_config = config_util.load_user_config(args.dataset_config)
+            ignored = ["train_data_dir", "reg_data_dir"]
+            if any(getattr(args, attr) is not None for attr in ignored):
+                print(
+                    "ignore following options because config file is found: {0} / 設定ファイルが利用されるため以下のオプションは無視されます: {0}".format(
+                        ", ".join(ignored)
+                    )
+                )
+        else:
+            user_config = {
+                "datasets": [
+                    {"subsets": config_util.generate_dreambooth_subsets_config_by_subdirs(args.train_data_dir, args.reg_data_dir)}
+                ]
+            }
+
+        blueprint = blueprint_generator.generate(user_config, args, tokenizer=tokenizer)
+        train_dataset_group = config_util.generate_dataset_group_by_blueprint(blueprint.dataset_group)
+    else:
+        train_dataset_group = train_util.load_arbitrary_dataset(args, tokenizer)
+
+    current_epoch = Value("i", 0)
+    current_step = Value("i", 0)
+    ds_for_collater = train_dataset_group if args.max_data_loader_n_workers == 0 else None
+    collater = train_util.collater_class(current_epoch, current_step, ds_for_collater)
+
+    if args.no_token_padding:
+        train_dataset_group.disable_token_padding()
+
+    if args.debug_dataset:
+        train_util.debug_dataset(train_dataset_group)
+        return
+
+    if cache_latents:
+        assert (
+            train_dataset_group.is_latent_cacheable()
+        ), "when caching latents, either color_aug or random_crop cannot be used / latentをキャッシュするときはcolor_augとrandom_cropは使えません"
+
+    # acceleratorを準備する
+    print("prepare accelerator")
+
+    if args.gradient_accumulation_steps > 1:
+        print(
+            f"gradient_accumulation_steps is {args.gradient_accumulation_steps}. accelerate does not support gradient_accumulation_steps when training multiple models (U-Net and Text Encoder), so something might be wrong"
+        )
+        print(
+            f"gradient_accumulation_stepsが{args.gradient_accumulation_steps}に設定されています。accelerateは複数モデル（U-NetおよびText Encoder）の学習時にgradient_accumulation_stepsをサポートしていないため結果は未知数です"
+        )
+
+    accelerator = train_util.prepare_accelerator(args)
+
+    # mixed precisionに対応した型を用意しておき適宜castする
+    weight_dtype, save_dtype = train_util.prepare_dtype(args)
+
+    # モデルを読み込む
+    text_encoder, vae, unet, load_stable_diffusion_format = train_util.load_target_model(args, weight_dtype, accelerator)
+
+    # verify load/save model formats
+    if load_stable_diffusion_format:
+        src_stable_diffusion_ckpt = args.pretrained_model_name_or_path
+        src_diffusers_model_path = None
+    else:
+        src_stable_diffusion_ckpt = None
+        src_diffusers_model_path = args.pretrained_model_name_or_path
+
+    if args.save_model_as is None:
+        save_stable_diffusion_format = load_stable_diffusion_format
+        use_safetensors = args.use_safetensors
+    else:
+        save_stable_diffusion_format = args.save_model_as.lower() == "ckpt" or args.save_model_as.lower() == "safetensors"
+        use_safetensors = args.use_safetensors or ("safetensors" in args.save_model_as.lower())
+
+    # モデルに xformers とか memory efficient attention を組み込む
+    train_util.replace_unet_modules(unet, args.mem_eff_attn, args.xformers, args.sdpa)
+
+    # 学習を準備する
+    if cache_latents:
+        vae.to(accelerator.device, dtype=weight_dtype)
+        vae.requires_grad_(False)
+        vae.eval()
+        with torch.no_grad():
+            train_dataset_group.cache_latents(vae, args.vae_batch_size, args.cache_latents_to_disk, accelerator.is_main_process)
+        vae.to("cpu")
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        gc.collect()
+
+        accelerator.wait_for_everyone()
+
+    # 学習を準備する：モデルを適切な状態にする
+    train_text_encoder = args.stop_text_encoder_training is None or args.stop_text_encoder_training >= 0
+    unet.requires_grad_(True)  # 念のため追加
+    text_encoder.requires_grad_(train_text_encoder)
+    if not train_text_encoder:
+        accelerator.print("Text Encoder is not trained.")
+
+    if args.gradient_checkpointing:
+        unet.enable_gradient_checkpointing()
+        text_encoder.gradient_checkpointing_enable()
+
+    if not cache_latents:
+        vae.requires_grad_(False)
+        vae.eval()
+        vae.to(accelerator.device, dtype=weight_dtype)
+
+    # 学習に必要なクラスを準備する
+    accelerator.print("prepare optimizer, data loader etc.")
+    if train_text_encoder:
+        trainable_params = itertools.chain(unet.parameters(), text_encoder.parameters())
+    else:
+        trainable_params = unet.parameters()
+
+    _, _, optimizer = train_util.get_optimizer(args, trainable_params)
+
+    # dataloaderを準備する
+    # DataLoaderのプロセス数：0はメインプロセスになる
+    n_workers = min(args.max_data_loader_n_workers, os.cpu_count() - 1)  # cpu_count-1 ただし最大で指定された数まで
+    train_dataloader = torch.utils.data.DataLoader(
+        train_dataset_group,
+        batch_size=1,
+        shuffle=True,
+        collate_fn=collater,
+        num_workers=n_workers,
+        persistent_workers=args.persistent_data_loader_workers,
+    )
+
+    # 学習ステップ数を計算する
+    if args.max_train_epochs is not None:
+        args.max_train_steps = args.max_train_epochs * math.ceil(
+            len(train_dataloader) / accelerator.num_processes / args.gradient_accumulation_steps
+        )
+        accelerator.print(f"override steps. steps for {args.max_train_epochs} epochs is / 指定エポックまでのステップ数: {args.max_train_steps}")
+
+    # データセット側にも学習ステップを送信
+    train_dataset_group.set_max_train_steps(args.max_train_steps)
+
+    if args.stop_text_encoder_training is None:
+        args.stop_text_encoder_training = args.max_train_steps + 1  # do not stop until end
+
+    # lr schedulerを用意する TODO gradient_accumulation_stepsの扱いが何かおかしいかもしれない。後で確認する
+    lr_scheduler = train_util.get_scheduler_fix(args, optimizer, accelerator.num_processes)
+
+    # 実験的機能：勾配も含めたfp16学習を行う　モデル全体をfp16にする
+    if args.full_fp16:
+        assert (
+            args.mixed_precision == "fp16"
+        ), "full_fp16 requires mixed precision='fp16' / full_fp16を使う場合はmixed_precision='fp16'を指定してください。"
+        accelerator.print("enable full fp16 training.")
+        unet.to(weight_dtype)
+        text_encoder.to(weight_dtype)
+
+    # acceleratorがなんかよろしくやってくれるらしい
+    if train_text_encoder:
+        unet, text_encoder, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+            unet, text_encoder, optimizer, train_dataloader, lr_scheduler
+        )
+    else:
+        unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(unet, optimizer, train_dataloader, lr_scheduler)
+
+    # transform DDP after prepare
+    text_encoder, unet = train_util.transform_if_model_is_DDP(text_encoder, unet)
+
+    if not train_text_encoder:
+        text_encoder.to(accelerator.device, dtype=weight_dtype)  # to avoid 'cpu' vs 'cuda' error
+
+    # 実験的機能：勾配も含めたfp16学習を行う　PyTorchにパッチを当ててfp16でのgrad scaleを有効にする
+    if args.full_fp16:
+        train_util.patch_accelerator_for_fp16_training(accelerator)
+
+    # resumeする
+    train_util.resume_from_local_or_hf_if_specified(accelerator, args)
+
+    # epoch数を計算する
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+    if (args.save_n_epoch_ratio is not None) and (args.save_n_epoch_ratio > 0):
+        args.save_every_n_epochs = math.floor(num_train_epochs / args.save_n_epoch_ratio) or 1
+
+    # 学習する
+    total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
+    accelerator.print("running training / 学習開始")
+    accelerator.print(f"  num train images * repeats / 学習画像の数×繰り返し回数: {train_dataset_group.num_train_images}")
+    accelerator.print(f"  num reg images / 正則化画像の数: {train_dataset_group.num_reg_images}")
+    accelerator.print(f"  num batches per epoch / 1epochのバッチ数: {len(train_dataloader)}")
+    accelerator.print(f"  num epochs / epoch数: {num_train_epochs}")
+    accelerator.print(f"  batch size per device / バッチサイズ: {args.train_batch_size}")
+    accelerator.print(
+        f"  total train batch size (with parallel & distributed & accumulation) / 総バッチサイズ（並列学習、勾配合計含む）: {total_batch_size}"
+    )
+    accelerator.print(f"  gradient ccumulation steps / 勾配を合計するステップ数 = {args.gradient_accumulation_steps}")
+    accelerator.print(f"  total optimization steps / 学習ステップ数: {args.max_train_steps}")
+
+    progress_bar = tqdm(range(args.max_train_steps), smoothing=0, disable=not accelerator.is_local_main_process, desc="steps")
+    global_step = 0
+
+    noise_scheduler = DDPMScheduler(
+        beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, clip_sample=False
+    )
+    prepare_scheduler_for_custom_training(noise_scheduler, accelerator.device)
+
+    if accelerator.is_main_process:
+        accelerator.init_trackers("dreambooth" if args.log_tracker_name is None else args.log_tracker_name)
+
+    loss_list = []
+    loss_total = 0.0
+    for epoch in range(num_train_epochs):
+        accelerator.print(f"\nepoch {epoch+1}/{num_train_epochs}")
+        current_epoch.value = epoch + 1
+
+        # 指定したステップ数までText Encoderを学習する：epoch最初の状態
+        unet.train()
+        # train==True is required to enable gradient_checkpointing
+        if args.gradient_checkpointing or global_step < args.stop_text_encoder_training:
+            text_encoder.train()
+
+        for step, batch in enumerate(train_dataloader):
+            current_step.value = global_step
+            # 指定したステップ数でText Encoderの学習を止める
+            if global_step == args.stop_text_encoder_training:
+                accelerator.print(f"stop text encoder training at step {global_step}")
+                if not args.gradient_checkpointing:
+                    text_encoder.train(False)
+                text_encoder.requires_grad_(False)
+
+            with accelerator.accumulate(unet):
+                with torch.no_grad():
+                    # latentに変換
+                    if cache_latents:
+                        latents = batch["latents"].to(accelerator.device)
+                    else:
+                        latents = vae.encode(batch["images"].to(dtype=weight_dtype)).latent_dist.sample()
+                    latents = latents * 0.18215
+                b_size = latents.shape[0]
+
+                # Get the text embedding for conditioning
+                with torch.set_grad_enabled(global_step < args.stop_text_encoder_training):
+                    if args.weighted_captions:
+                        encoder_hidden_states = get_weighted_text_embeddings(
+                            tokenizer,
+                            text_encoder,
+                            batch["captions"],
+                            accelerator.device,
+                            args.max_token_length // 75 if args.max_token_length else 1,
+                            clip_skip=args.clip_skip,
+                        )
+                    else:
+                        input_ids = batch["input_ids"].to(accelerator.device)
+                        encoder_hidden_states = train_util.get_hidden_states(
+                            args, input_ids, tokenizer, text_encoder, None if not args.full_fp16 else weight_dtype
+                        )
+
+                # Sample noise, sample a random timestep for each image, and add noise to the latents,
+                # with noise offset and/or multires noise if specified
+                noise, noisy_latents, timesteps = train_util.get_noise_noisy_latents_and_timesteps(args, noise_scheduler, latents)
+
+                # Predict the noise residual
+                with accelerator.autocast():
+                    noise_pred = unet(noisy_latents, timesteps, encoder_hidden_states).sample
+
+                if args.v_parameterization:
+                    # v-parameterization training
+                    target = noise_scheduler.get_velocity(latents, noise, timesteps)
+                else:
+                    target = noise
+
+                loss = torch.nn.functional.mse_loss(noise_pred.float(), target.float(), reduction="none")
+                loss = loss.mean([1, 2, 3])
+
+                loss_weights = batch["loss_weights"]  # 各sampleごとのweight
+                loss = loss * loss_weights
+
+                if args.min_snr_gamma:
+                    loss = apply_snr_weight(loss, timesteps, noise_scheduler, args.min_snr_gamma)
+                if args.scale_v_pred_loss_like_noise_pred:
+                    loss = scale_v_prediction_loss_like_noise_prediction(loss, timesteps, noise_scheduler)
+
+                loss = loss.mean()  # 平均なのでbatch_sizeで割る必要なし
+
+                accelerator.backward(loss)
+                if accelerator.sync_gradients and args.max_grad_norm != 0.0:
+                    if train_text_encoder:
+                        params_to_clip = itertools.chain(unet.parameters(), text_encoder.parameters())
+                    else:
+                        params_to_clip = unet.parameters()
+                    accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
+
+                optimizer.step()
+                lr_scheduler.step()
+                optimizer.zero_grad(set_to_none=True)
+
+            # Checks if the accelerator has performed an optimization step behind the scenes
+            if accelerator.sync_gradients:
+                progress_bar.update(1)
+                global_step += 1
+
+                train_util.sample_images(
+                    accelerator, args, None, global_step, accelerator.device, vae, tokenizer, text_encoder, unet
+                )
+
+                # 指定ステップごとにモデルを保存
+                if args.save_every_n_steps is not None and global_step % args.save_every_n_steps == 0:
+                    accelerator.wait_for_everyone()
+                    if accelerator.is_main_process:
+                        src_path = src_stable_diffusion_ckpt if save_stable_diffusion_format else src_diffusers_model_path
+                        train_util.save_sd_model_on_epoch_end_or_stepwise(
+                            args,
+                            False,
+                            accelerator,
+                            src_path,
+                            save_stable_diffusion_format,
+                            use_safetensors,
+                            save_dtype,
+                            epoch,
+                            num_train_epochs,
+                            global_step,
+                            accelerator.unwrap_model(text_encoder),
+                            accelerator.unwrap_model(unet),
+                            vae,
+                        )
+
+            current_loss = loss.detach().item()
+            if args.logging_dir is not None:
+                logs = {"loss": current_loss, "lr": float(lr_scheduler.get_last_lr()[0])}
+                if (
+                    args.optimizer_type.lower().startswith("DAdapt".lower()) or args.optimizer_type.lower() == "Prodigy".lower()
+                ):  # tracking d*lr value
+                    logs["lr/d*lr"] = (
+                        lr_scheduler.optimizers[0].param_groups[0]["d"] * lr_scheduler.optimizers[0].param_groups[0]["lr"]
+                    )
+                accelerator.log(logs, step=global_step)
+
+            if epoch == 0:
+                loss_list.append(current_loss)
+            else:
+                loss_total -= loss_list[step]
+                loss_list[step] = current_loss
+            loss_total += current_loss
+            avr_loss = loss_total / len(loss_list)
+            logs = {"loss": avr_loss}  # , "lr": lr_scheduler.get_last_lr()[0]}
+            progress_bar.set_postfix(**logs)
+
+            if global_step >= args.max_train_steps:
+                break
+
+        if args.logging_dir is not None:
+            logs = {"loss/epoch": loss_total / len(loss_list)}
+            accelerator.log(logs, step=epoch + 1)
+
+        accelerator.wait_for_everyone()
+
+        if args.save_every_n_epochs is not None:
+            if accelerator.is_main_process:
+                # checking for saving is in util
+                src_path = src_stable_diffusion_ckpt if save_stable_diffusion_format else src_diffusers_model_path
+                train_util.save_sd_model_on_epoch_end_or_stepwise(
+                    args,
+                    True,
+                    accelerator,
+                    src_path,
+                    save_stable_diffusion_format,
+                    use_safetensors,
+                    save_dtype,
+                    epoch,
+                    num_train_epochs,
+                    global_step,
+                    accelerator.unwrap_model(text_encoder),
+                    accelerator.unwrap_model(unet),
+                    vae,
+                )
+
+        train_util.sample_images(accelerator, args, epoch + 1, global_step, accelerator.device, vae, tokenizer, text_encoder, unet)
+
+    is_main_process = accelerator.is_main_process
+    if is_main_process:
+        unet = accelerator.unwrap_model(unet)
+        text_encoder = accelerator.unwrap_model(text_encoder)
+
+    accelerator.end_training()
+
+    if args.save_state and is_main_process:
+        train_util.save_state_on_train_end(args, accelerator)
+
+    del accelerator  # この後メモリを使うのでこれは消す
+
+    if is_main_process:
+        src_path = src_stable_diffusion_ckpt if save_stable_diffusion_format else src_diffusers_model_path
+        train_util.save_sd_model_on_train_end(
+            args, src_path, save_stable_diffusion_format, use_safetensors, save_dtype, epoch, global_step, text_encoder, unet, vae
+        )
+        print("model saved.")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    train_util.add_sd_models_arguments(parser)
+    train_util.add_dataset_arguments(parser, True, False, True)
+    train_util.add_training_arguments(parser, True)
+    train_util.add_sd_saving_arguments(parser)
+    train_util.add_optimizer_arguments(parser)
+    config_util.add_config_arguments(parser)
+    custom_train_functions.add_custom_train_arguments(parser)
+
+    parser.add_argument(
+        "--no_token_padding",
+        action="store_true",
+        help="disable token padding (same as Diffuser's DreamBooth) / トークンのpaddingを無効にする（Diffusers版DreamBoothと同じ動作）",
+    )
+    parser.add_argument(
+        "--stop_text_encoder_training",
+        type=int,
+        default=None,
+        help="steps to stop text encoder training, -1 for no training / Text Encoderの学習を止めるステップ数、-1で最初から学習しない",
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    args = train_util.read_config_from_file(args, parser)
+
+    train(args)
diff --git a/train_db_README.md b/train_db_README.md
new file mode 100644
index 0000000000000000000000000000000000000000..2367d29ae3180e4e92d10943baa9aeafd9ad4e8b
--- /dev/null
+++ b/train_db_README.md
@@ -0,0 +1,295 @@
+A guide to DreamBooth. The same procedure is used for training additional networks such as LoRA.
+
+# overview
+
+The main functions of the script are as follows.
+
+- Memory saving by 8bit Adam optimizer and latent cache (similar to ShivamShirao's version).
+- Saved memory by xformers.
+- Study in any size, not just 512x512.
+- Quality improvement with augmentation.
+- Supports fine tuning of Text Encoder+U-Net as well as DreamBooth.
+- Read and write models in StableDiffusion format.
+- Aspect Ratio Bucketing.
+- Supports Stable Diffusion v2.0.
+
+# learning procedure
+
+## step 1. Environment improvement
+
+See the README in this repository.
+
+
+## step 2. Determine identifier and class
+
+Decide the word identifier that connects the target you want to learn and the class to which the target belongs.
+
+(There are various names such as instance, but for the time being I will stick to the original paper.)
+
+Here's a very brief explanation (look it up for more details).
+
+class is the general type to learn. For example, if you want to learn a specific breed of dog, the class will be dog. Anime characters will be boy, girl, 1boy or 1girl depending on the model.
+
+The identifier is for identifying and learning the learning target. Any word is fine, but according to the original paper, ``a rare word with 3 letters or less that becomes one token with tokinizer'' is good.
+
+By using the identifier and class to train the model, for example, "shs dog", you can learn by identifying the object you want to learn from the class.
+
+When generating an image, if you say "shs dog", an image of the learned dog breed will be generated.
+
+(For reference, the identifier I use these days is ``shs sts scs cpc coc cic msm usu ici lvl cic dii muk ori hru rik koo yos wny``.)
+
+## step 3. Prepare images for training
+Create a folder to store training images. __In addition, create a directory with the following name:
+
+```
+<repeat count>_<identifier> <class>
+```
+
+Don't forget the ``_`` between them.
+
+The number of repetitions is specified to match the number of regularized images (described later).
+
+For example, at the prompt "sls frog", to repeat the data 20 times, it would be "20_sls frog". It will be as follows.
+
+![image](https://user-images.githubusercontent.com/52813779/210770636-1c851377-5936-4c15-90b7-8ac8ad6c2074.png)
+
+## step 4. Preparing regularized images
+This is the procedure when using a regularized image. It is also possible to learn without using the regularization image (the whole target class is affected because it is impossible to distinguish without using the regularization image).
+
+Create a folder to store the regularized images. __In addition, __ create a directory named ``<repeat count>_<class>``.
+
+For example, with the prompt "frog" and without repeating the data (just once):
+
+![image](https://user-images.githubusercontent.com/52813779/210770897-329758e5-3675-49f1-b345-c135f1725832.png)
+
+Specify the number of iterations so that " __ number of iterations of training images x number of training images ≥ number of iterations of regularization images x number of regularization images __".
+
+(The number of data in one epoch is "number of repetitions of training images x number of training images". If the number of regularization images is more than that, the remaining regularization images will not be used.)
+
+## step 5. Run training
+Run the script. The maximally memory-saving command looks like this (actually typed on one line):
+
+*The command for learning additional networks such as LoRA is ``train_network.py`` instead of ``train_db.py``. You will also need additional network_\* options, so please refer to LoRA's guide.
+
+```
+accelerate launch --num_cpu_threads_per_process 8 train_db.py
+     --pretrained_model_name_or_path=<directory of .ckpt or .safetensord or Diffusers model>
+     --train_data_dir=<training data directory>
+     --reg_data_dir=<regularized image directory>
+     --output_dir=<output destination directory for trained model>
+     --prior_loss_weight=1.0
+     --resolution=512
+     --train_batch_size=1
+     --learning_rate=1e-6
+     --max_train_steps=1600
+     --use_8bit_adam
+     --xformers
+     --mixed_precision="bf16"
+     --cache_latents
+     --gradient_checkpointing
+```
+
+It seems to be good to specify the number of CPU cores for num_cpu_threads_per_process.
+
+Specify the model to perform additional training in pretrained_model_name_or_path. You can specify a Stable Diffusion checkpoint file (.ckpt or .safetensors), a model directory on the Diffusers local disk, or a Diffusers model ID (such as "stabilityai/stable-diffusion-2"). The saved model after training will be saved in the same format as the original model by default (can be changed with the save_model_as option).
+
+prior_loss_weight is the loss weight of the regularized image. Normally, specify 1.0.
+
+resolution will be the size of the image (resolution, width and height). If bucketing (described later) is not used, use this size for training images and regularization images.
+
+train_batch_size is the training batch size. Set max_train_steps to 1600. The learning rate learning_rate is 5e-6 in the diffusers version and 1e-6 in the StableDiffusion version, so 1e-6 is specified here.
+
+Specify mixed_precision="bf16" (or "fp16") and gradient_checkpointing for memory saving.
+
+Specify the xformers option and use xformers' CrossAttention. If you don't have xformers installed, if you get an error (without mixed_precision, it was an error in my environment), specify the mem_eff_attn option instead to use the memory-saving version of CrossAttention (speed will be slower) .
+
+Cache VAE output with cache_latents option to save memory.
+
+If you have a certain amount of memory, specify it as follows, for example.
+
+```
+accelerate launch --num_cpu_threads_per_process 8 train_db.py
+     --pretrained_model_name_or_path=<directory of .ckpt or .safetensord or Diffusers model>
+     --train_data_dir=<training data directory>
+     --reg_data_dir=<regularized image directory>
+     --output_dir=<output destination directory for trained model>
+     --prior_loss_weight=1.0
+     --resolution=512
+     --train_batch_size=4
+     --learning_rate=1e-6
+     --max_train_steps=400
+     --use_8bit_adam
+     --xformers
+     --mixed_precision="bf16"
+     --cache_latents
+```
+
+Remove gradient_checkpointing to speed up (memory usage will increase). Increase the batch size to improve speed and accuracy.
+
+An example of using bucketing (see below) and using augmentation (see below) looks like this:
+
+```
+accelerate launch --num_cpu_threads_per_process 8 train_db.py
+     --pretrained_model_name_or_path=<directory of .ckpt or .safetensord or Diffusers model>
+     --train_data_dir=<training data directory>
+     --reg_data_dir=<regularized image directory>
+     --output_dir=<output destination directory for trained model>
+     --resolution=768,512
+     --train_batch_size=20 --learning_rate=5e-6 --max_train_steps=800
+     --use_8bit_adam --xformers --mixed_precision="bf16"
+     --save_every_n_epochs=1 --save_state --save_precision="bf16"
+     --logging_dir=logs
+     --enable_bucket --min_bucket_reso=384 --max_bucket_reso=1280
+     --color_aug --flip_aug --gradient_checkpointing --seed 42
+```
+
+### About the number of steps
+To save memory, the number of training steps per step is half that of train_drebooth.py (because the target image and the regularization image are divided into different batches instead of the same batch).
+Double the number of steps to get almost the same training as the original Diffusers version and XavierXiao's StableDiffusion version.
+
+(Strictly speaking, the order of the data changes due to shuffle=True, but I don't think it has a big impact on learning.)
+
+## Generate an image with the trained model
+
+Name last.ckpt in the specified folder when learning is completed will output the checkpoint (if you learned the DiffUsers version model, it will be the last folder).
+
+For v1.4/1.5 and other derived models, this model can be inferred by Automatic1111's WebUI, etc. Place it in the models\Stable-diffusion folder.
+
+When generating images with WebUI with the v2.x model, a separate .yaml file that describes the model specifications is required. Place v2-inference.yaml for v2.x base and v2-inference-v.yaml for 768/v in the same folder and make the part before the extension the same name as the model.
+
+![image](https://user-images.githubusercontent.com/52813779/210776915-061d79c3-6582-42c2-8884-8b91d2f07313.png)
+
+Each yaml file can be found at [https://github.com/Stability-AI/stablediffusion/tree/main/configs/stable-diffusion] (Stability AI SD2.0 repository).
+
+# Other study options
+
+## Supports Stable Diffusion 2.0 --v2 / --v_parameterization
+Specify the v2 option when using Hugging Face's stable-diffusion-2-base, and specify both the v2 and v_parameterization options when using stable-diffusion-2 or 768-v-ema.ckpt.
+
+In addition, learning SD 2.0 seems to be difficult with VRAM 12GB because the Text Encoder is getting bigger.
+
+The following points have changed significantly in Stable Diffusion 2.0.
+
+1. Tokenizer to use
+2. Which Text Encoder to use and which output layer to use (2.0 uses the penultimate layer)
+3. Output dimensionality of Text Encoder (768->1024)
+4. Structure of U-Net (number of heads of CrossAttention, etc.)
+5. v-parameterization (the sampling method seems to have changed)
+
+Among these, 1 to 4 are adopted for base, and 1 to 5 are adopted for the one without base (768-v). Enabling 1-4 is the v2 option, and enabling 5 is the v_parameterization option.
+
+## check training data --debug_dataset
+By adding this option, you can check what kind of image data and captions will be learned in advance before learning. Press Esc to exit and return to the command line.
+
+*Please note that it seems to hang when executed in an environment where there is no screen such as Colab.
+
+## Stop training Text Encoder --stop_text_encoder_training
+If you specify a numerical value for the stop_text_encoder_training option, after that number of steps, only the U-Net will be trained without training the Text Encoder. In some cases, the accuracy may be improved.
+
+(Probably only the Text Encoder may overfit first, and I guess that it can be prevented, but the detailed impact is unknown.)
+
+## Load and learn VAE separately --vae
+If you specify either a Stable Diffusion checkpoint, a VAE checkpoint file, a Diffuses model, or a VAE (both of which can specify a local or Hugging Face model ID) in the vae option, that VAE is used for learning (latents when caching or getting latents during learning).
+The saved model will incorporate this VAE.
+
+## save during learning --save_every_n_epochs / --save_state / --resume
+Specifying a number for the save_every_n_epochs option saves the model during training every epoch.
+
+If you specify the save_state option at the same time, the learning state including the state of the optimizer etc. will be saved together (compared to restarting learning from the checkpoint, you can expect to improve accuracy and shorten the learning time). The learning state is output in a folder named "epoch-??????-state" (?????? is the number of epochs) in the destination folder. Please use it when studying for a long time.
+
+Use the resume option to resume training from a saved training state. Please specify the learning state folder.
+
+Please note that due to the specifications of Accelerator (?), the number of epochs and global step are not saved, and it will start from 1 even when you resume.
+
+## No tokenizer padding --no_token_padding
+The no_token_padding option does not pad the output of the Tokenizer (same behavior as Diffusers version of old DreamBooth).
+
+## Training with arbitrary size images --resolution
+You can study outside the square. Please specify "width, height" like "448,640" in resolution. Width and height must be divisible by 64. Match the size of the training image and the regularization image.
+
+Personally, I often generate vertically long images, so I sometimes learn with "448, 640".
+
+## Aspect Ratio Bucketing --enable_bucket / --min_bucket_reso / --max_bucket_reso
+It is enabled by specifying the enable_bucket option. Stable Diffusion is trained at 512x512, but also at resolutions such as 256x768 and 384x640.
+
+If you specify this option, you do not need to unify the training images and regularization images to a specific resolution. Choose from several resolutions (aspect ratios) and learn at that resolution.
+Since the resolution is 64 pixels, the aspect ratio may not be exactly the same as the original image.
+
+You can specify the minimum size of the resolution with the min_bucket_reso option and the maximum size with the max_bucket_reso. The defaults are 256 and 1024 respectively.
+For example, specifying a minimum size of 384 will not use resolutions such as 256x1024 or 320x768.
+If you increase the resolution to 768x768, you may want to specify 1280 as the maximum size.
+
+When Aspect Ratio Bucketing is enabled, it may be better to prepare regularization images with various resolutions that are similar to the training images.
+
+(Because the images in one batch are not biased toward training images and regularization images.
+
+## augmentation --color_aug / --flip_aug
+Augmentation is a method of improving model performance by dynamically changing data during learning. Learn while subtly changing the hue with color_aug and flipping left and right with flip_aug.
+
+Since the data changes dynamically, it cannot be specified together with the cache_latents option.
+
+## Specify data precision when saving --save_precision
+Specifying float, fp16, or bf16 as the save_precision option will save the checkpoint in that format (only when saving in Stable Diffusion format). Please use it when you want to reduce the size of checkpoint.
+
+## save in any format --save_model_as
+Specify the save format of the model. Specify one of ckpt, safetensors, diffusers, diffusers_safetensors.
+
+When reading Stable Diffusion format (ckpt or safetensors) and saving in Diffusers format, missing information is supplemented by dropping v1.5 or v2.1 information from Hugging Face.
+
+## Save learning log --logging_dir / --log_prefix
+Specify the log save destination folder in the logging_dir option. Logs in TensorBoard format are saved.
+
+For example, if you specify --logging_dir=logs, a logs folder will be created in your working folder, and logs will be saved in the date/time folder.
+Also, if you specify the --log_prefix option, the specified string will be added before the date and time. Use "--logging_dir=logs --log_prefix=db_style1_" for identification.
+
+To check the log with TensorBoard, open another command prompt and enter the following in the working folder (I think tensorboard is installed when Diffusers is installed, but if it is not installed, pip install Please put it in tensorboard).
+
+```
+tensorboard --logdir=logs
+```
+
+Then open your browser and go to http://localhost:6006/ to see it.
+
+## scheduler related specification of learning rate --lr_scheduler / --lr_warmup_steps
+You can choose the learning rate scheduler from linear, cosine, cosine_with_restarts, polynomial, constant, constant_with_warmup with the lr_scheduler option. Default is constant. With lr_warmup_steps, you can specify the number of steps to warm up the scheduler (gradually changing the learning rate). Please do your own research for details.
+
+## Training with fp16 gradient (experimental feature) --full_fp16
+The full_fp16 option will change the gradient from normal float32 to float16 (fp16) and learn (it seems to be full fp16 learning instead of mixed precision).
+As a result, it seems that the SD1.x 512x512 size can be learned with a VRAM usage of less than 8GB, and the SD2.x 512x512 size can be learned with a VRAM usage of less than 12GB.
+
+Specify fp16 in the accelerate config beforehand and optionally set ``mixed_precision="fp16"`` (bf16 does not work).
+
+To minimize memory usage, use xformers, use_8bit_adam, cache_latents, gradient_checkpointing options and set train_batch_size to 1.
+
+(If you can afford it, increasing the train_batch_size step by step should improve the accuracy a little.)
+
+It is realized by patching the PyTorch source (confirmed with PyTorch 1.12.1 and 1.13.0). Accuracy will drop considerably, and the probability of learning failure on the way will also increase.
+The setting of the learning rate and the number of steps seems to be severe. Please be aware of them and use them at your own risk.
+
+# Other learning methods
+
+## Learning multiple classes, multiple identifiers
+The method is simple, multiple folders with ``Repetition count_<identifier> <class>`` in the training image folder, and a folder with ``Repetition count_<class>`` in the regularization image folder. Please prepare multiple
+
+For example, learning "sls frog" and "cpc rabbit" at the same time would look like this:
+
+![image](https://user-images.githubusercontent.com/52813779/210777933-a22229db-b219-4cd8-83ca-e87320fc4192.png)
+
+If you have one class and multiple targets, you can have only one regularized image folder. For example, if 1girl has character A and character B, do as follows.
+
+- train_girls
+   - 10_sls 1girl
+   - 10_cpc 1girl
+- reg_girls
+   -1_1girl
+
+If the number of data varies, it seems that good results can be obtained by adjusting the number of repetitions to unify the number of sheets for each class and identifier.
+
+## Use captions in DreamBooth
+If you put a file with the same file name as the image and the extension .caption (you can change it in the option) in the training image and regularization image folders, the caption will be read from that file and learned as a prompt.
+
+* The folder name (identifier class) will no longer be used for training those images.
+
+Adding captions to each image (you can use BLIP, etc.) may help clarify the attributes you want to learn.
+
+Caption files have a .caption extension by default. You can change it with --caption_extension. With the --shuffle_caption option, study captions during learning while shuffling each part separated by commas.
\ No newline at end of file
diff --git a/train_network.py b/train_network.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7ee451b13d91b4972ee3d0f2cb78268233e3d2e
--- /dev/null
+++ b/train_network.py
@@ -0,0 +1,974 @@
+import importlib
+import argparse
+import gc
+import math
+import os
+import sys
+import random
+import time
+import json
+from multiprocessing import Value
+
+from tqdm import tqdm
+import torch
+from accelerate.utils import set_seed
+from diffusers import DDPMScheduler
+from library import model_util
+
+import library.train_util as train_util
+from library.train_util import (
+    DreamBoothDataset,
+)
+import library.config_util as config_util
+from library.config_util import (
+    ConfigSanitizer,
+    BlueprintGenerator,
+)
+import library.huggingface_util as huggingface_util
+import library.custom_train_functions as custom_train_functions
+from library.custom_train_functions import (
+    apply_snr_weight,
+    get_weighted_text_embeddings,
+    prepare_scheduler_for_custom_training,
+    pyramid_noise_like,
+    apply_noise_offset,
+    scale_v_prediction_loss_like_noise_prediction,
+)
+
+
+class NetworkTrainer:
+    def __init__(self):
+        self.vae_scale_factor = 0.18215
+
+    # TODO 他のスクリプトと共通化する
+    def generate_step_logs(
+        self, args: argparse.Namespace, current_loss, avr_loss, lr_scheduler, keys_scaled=None, mean_norm=None, maximum_norm=None
+    ):
+        logs = {"loss/current": current_loss, "loss/average": avr_loss}
+
+        if keys_scaled is not None:
+            logs["max_norm/keys_scaled"] = keys_scaled
+            logs["max_norm/average_key_norm"] = mean_norm
+            logs["max_norm/max_key_norm"] = maximum_norm
+
+        lrs = lr_scheduler.get_last_lr()
+
+        if args.network_train_text_encoder_only or len(lrs) <= 2:  # not block lr (or single block)
+            if args.network_train_unet_only:
+                logs["lr/unet"] = float(lrs[0])
+            elif args.network_train_text_encoder_only:
+                logs["lr/textencoder"] = float(lrs[0])
+            else:
+                logs["lr/textencoder"] = float(lrs[0])
+                logs["lr/unet"] = float(lrs[-1])  # may be same to textencoder
+
+            if (
+                args.optimizer_type.lower().startswith("DAdapt".lower()) or args.optimizer_type.lower() == "Prodigy".lower()
+            ):  # tracking d*lr value of unet.
+                logs["lr/d*lr"] = (
+                    lr_scheduler.optimizers[-1].param_groups[0]["d"] * lr_scheduler.optimizers[-1].param_groups[0]["lr"]
+                )
+        else:
+            idx = 0
+            if not args.network_train_unet_only:
+                logs["lr/textencoder"] = float(lrs[0])
+                idx = 1
+
+            for i in range(idx, len(lrs)):
+                logs[f"lr/group{i}"] = float(lrs[i])
+                if args.optimizer_type.lower().startswith("DAdapt".lower()) or args.optimizer_type.lower() == "Prodigy".lower():
+                    logs[f"lr/d*lr/group{i}"] = (
+                        lr_scheduler.optimizers[-1].param_groups[i]["d"] * lr_scheduler.optimizers[-1].param_groups[i]["lr"]
+                    )
+
+        return logs
+
+    def assert_extra_args(self, args, train_dataset_group):
+        pass
+
+    def load_target_model(self, args, weight_dtype, accelerator):
+        text_encoder, vae, unet, _ = train_util.load_target_model(args, weight_dtype, accelerator)
+        return model_util.get_model_version_str_for_sd1_sd2(args.v2, args.v_parameterization), text_encoder, vae, unet
+
+    def load_tokenizer(self, args):
+        tokenizer = train_util.load_tokenizer(args)
+        return tokenizer
+
+    def is_text_encoder_outputs_cached(self, args):
+        return False
+
+    def cache_text_encoder_outputs_if_needed(
+        self, args, accelerator, unet, vae, tokenizers, text_encoders, data_loader, weight_dtype
+    ):
+        for t_enc in text_encoders:
+            t_enc.to(accelerator.device)
+
+    def get_text_cond(self, args, accelerator, batch, tokenizers, text_encoders, weight_dtype):
+        input_ids = batch["input_ids"].to(accelerator.device)
+        encoder_hidden_states = train_util.get_hidden_states(args, input_ids, tokenizers[0], text_encoders[0], weight_dtype)
+        return encoder_hidden_states
+
+    def call_unet(self, args, accelerator, unet, noisy_latents, timesteps, text_conds, batch, weight_dtype):
+        noise_pred = unet(noisy_latents, timesteps, text_conds).sample
+        return noise_pred
+
+    def sample_images(self, accelerator, args, epoch, global_step, device, vae, tokenizer, text_encoder, unet):
+        train_util.sample_images(accelerator, args, epoch, global_step, device, vae, tokenizer, text_encoder, unet)
+
+    def train(self, args):
+        session_id = random.randint(0, 2**32)
+        training_started_at = time.time()
+        train_util.verify_training_args(args)
+        train_util.prepare_dataset_args(args, True)
+
+        cache_latents = args.cache_latents
+        use_dreambooth_method = args.in_json is None
+        use_user_config = args.dataset_config is not None
+
+        if args.seed is None:
+            args.seed = random.randint(0, 2**32)
+        set_seed(args.seed)
+
+        # tokenizerは単体またはリスト、tokenizersは必ずリスト：既存のコードとの互換性のため
+        tokenizer = self.load_tokenizer(args)
+        tokenizers = tokenizer if isinstance(tokenizer, list) else [tokenizer]
+
+        # データセットを準備する
+        if args.dataset_class is None:
+            blueprint_generator = BlueprintGenerator(ConfigSanitizer(True, True, False, True))
+            if use_user_config:
+                print(f"Loading dataset config from {args.dataset_config}")
+                user_config = config_util.load_user_config(args.dataset_config)
+                ignored = ["train_data_dir", "reg_data_dir", "in_json"]
+                if any(getattr(args, attr) is not None for attr in ignored):
+                    print(
+                        "ignoring the following options because config file is found: {0} / 設定ファイルが利用されるため以下のオプションは無視されます: {0}".format(
+                            ", ".join(ignored)
+                        )
+                    )
+            else:
+                if use_dreambooth_method:
+                    print("Using DreamBooth method.")
+                    user_config = {
+                        "datasets": [
+                            {
+                                "subsets": config_util.generate_dreambooth_subsets_config_by_subdirs(
+                                    args.train_data_dir, args.reg_data_dir
+                                )
+                            }
+                        ]
+                    }
+                else:
+                    print("Training with captions.")
+                    user_config = {
+                        "datasets": [
+                            {
+                                "subsets": [
+                                    {
+                                        "image_dir": args.train_data_dir,
+                                        "metadata_file": args.in_json,
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+
+            blueprint = blueprint_generator.generate(user_config, args, tokenizer=tokenizer)
+            train_dataset_group = config_util.generate_dataset_group_by_blueprint(blueprint.dataset_group)
+        else:
+            # use arbitrary dataset class
+            train_dataset_group = train_util.load_arbitrary_dataset(args, tokenizer)
+
+        current_epoch = Value("i", 0)
+        current_step = Value("i", 0)
+        ds_for_collater = train_dataset_group if args.max_data_loader_n_workers == 0 else None
+        collater = train_util.collater_class(current_epoch, current_step, ds_for_collater)
+
+        if args.debug_dataset:
+            train_util.debug_dataset(train_dataset_group)
+            return
+        if len(train_dataset_group) == 0:
+            print(
+                "No data found. Please verify arguments (train_data_dir must be the parent of folders with images) / 画像がありません。引数指定を確認してください（train_data_dirには画像があるフォルダではなく、画像があるフォルダの親フォルダを指定する必要があります）"
+            )
+            return
+
+        if cache_latents:
+            assert (
+                train_dataset_group.is_latent_cacheable()
+            ), "when caching latents, either color_aug or random_crop cannot be used / latentをキャッシュするときはcolor_augとrandom_cropは使えません"
+
+        self.assert_extra_args(args, train_dataset_group)
+
+        # acceleratorを準備する
+        print("preparing accelerator")
+        accelerator = train_util.prepare_accelerator(args)
+        is_main_process = accelerator.is_main_process
+
+        # mixed precisionに対応した型を用意しておき適宜castする
+        weight_dtype, save_dtype = train_util.prepare_dtype(args)
+        vae_dtype = torch.float32 if args.no_half_vae else weight_dtype
+
+        # モデルを読み込む
+        model_version, text_encoder, vae, unet = self.load_target_model(args, weight_dtype, accelerator)
+
+        # text_encoder is List[CLIPTextModel] or CLIPTextModel
+        text_encoders = text_encoder if isinstance(text_encoder, list) else [text_encoder]
+
+        # モデルに xformers とか memory efficient attention を組み込む
+        train_util.replace_unet_modules(unet, args.mem_eff_attn, args.xformers, args.sdpa)
+        vae.set_use_memory_efficient_attention_xformers(args.xformers)
+
+        # 差分追加学習のためにモデルを読み込む
+        sys.path.append(os.path.dirname(__file__))
+        accelerator.print("import network module:", args.network_module)
+        network_module = importlib.import_module(args.network_module)
+
+        if args.base_weights is not None:
+            # base_weights が指定されている場合は、指定された重みを読み込みマージする
+            for i, weight_path in enumerate(args.base_weights):
+                if args.base_weights_multiplier is None or len(args.base_weights_multiplier) <= i:
+                    multiplier = 1.0
+                else:
+                    multiplier = args.base_weights_multiplier[i]
+
+                accelerator.print(f"merging module: {weight_path} with multiplier {multiplier}")
+
+                module, weights_sd = network_module.create_network_from_weights(
+                    multiplier, weight_path, vae, text_encoder, unet, for_inference=True
+                )
+                module.merge_to(text_encoder, unet, weights_sd, weight_dtype, accelerator.device if args.lowram else "cpu")
+
+            accelerator.print(f"all weights merged: {', '.join(args.base_weights)}")
+
+        # 学習を準備する
+        if cache_latents:
+            vae.to(accelerator.device, dtype=vae_dtype)
+            vae.requires_grad_(False)
+            vae.eval()
+            with torch.no_grad():
+                train_dataset_group.cache_latents(vae, args.vae_batch_size, args.cache_latents_to_disk, accelerator.is_main_process)
+            vae.to("cpu")
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+            gc.collect()
+
+            accelerator.wait_for_everyone()
+
+        # 必要ならテキストエンコーダーの出力をキャッシュする: Text Encoderはcpuまたはgpuへ移される
+        self.cache_text_encoder_outputs_if_needed(
+            args, accelerator, unet, vae, tokenizers, text_encoders, train_dataset_group, weight_dtype
+        )
+
+        # prepare network
+        net_kwargs = {}
+        if args.network_args is not None:
+            for net_arg in args.network_args:
+                key, value = net_arg.split("=")
+                net_kwargs[key] = value
+
+        # if a new network is added in future, add if ~ then blocks for each network (;'∀')
+        if args.dim_from_weights:
+            network, _ = network_module.create_network_from_weights(1, args.network_weights, vae, text_encoder, unet, **net_kwargs)
+        else:
+            # LyCORIS will work with this...
+            network = network_module.create_network(
+                1.0,
+                args.network_dim,
+                args.network_alpha,
+                vae,
+                text_encoder,
+                unet,
+                neuron_dropout=args.network_dropout,
+                **net_kwargs,
+            )
+        if network is None:
+            return
+
+        if hasattr(network, "prepare_network"):
+            network.prepare_network(args)
+        if args.scale_weight_norms and not hasattr(network, "apply_max_norm_regularization"):
+            print(
+                "warning: scale_weight_norms is specified but the network does not support it / scale_weight_normsが指定されていますが、ネットワークが対応していません"
+            )
+            args.scale_weight_norms = False
+
+        train_unet = not args.network_train_text_encoder_only
+        train_text_encoder = not args.network_train_unet_only and not self.is_text_encoder_outputs_cached(args)
+        network.apply_to(text_encoder, unet, train_text_encoder, train_unet)
+
+        if args.network_weights is not None:
+            info = network.load_weights(args.network_weights)
+            accelerator.print(f"load network weights from {args.network_weights}: {info}")
+
+        if args.gradient_checkpointing:
+            unet.enable_gradient_checkpointing()
+            for t_enc in text_encoders:
+                t_enc.gradient_checkpointing_enable()
+            del t_enc
+            network.enable_gradient_checkpointing()  # may have no effect
+
+        # 学習に必要なクラスを準備する
+        accelerator.print("prepare optimizer, data loader etc.")
+
+        # 後方互換性を確保するよ
+        try:
+            trainable_params = network.prepare_optimizer_params(args.text_encoder_lr, args.unet_lr, args.learning_rate)
+        except TypeError:
+            accelerator.print(
+                "Deprecated: use prepare_optimizer_params(text_encoder_lr, unet_lr, learning_rate) instead of prepare_optimizer_params(text_encoder_lr, unet_lr)"
+            )
+            trainable_params = network.prepare_optimizer_params(args.text_encoder_lr, args.unet_lr)
+
+        optimizer_name, optimizer_args, optimizer = train_util.get_optimizer(args, trainable_params)
+
+        # dataloaderを準備する
+        # DataLoaderのプロセス数：0はメインプロセスになる
+        n_workers = min(args.max_data_loader_n_workers, os.cpu_count() - 1)  # cpu_count-1 ただし最大で指定された数まで
+
+        train_dataloader = torch.utils.data.DataLoader(
+            train_dataset_group,
+            batch_size=1,
+            shuffle=True,
+            collate_fn=collater,
+            num_workers=n_workers,
+            persistent_workers=args.persistent_data_loader_workers,
+        )
+
+        # 学習ステップ数を計算する
+        if args.max_train_epochs is not None:
+            args.max_train_steps = args.max_train_epochs * math.ceil(
+                len(train_dataloader) / accelerator.num_processes / args.gradient_accumulation_steps
+            )
+            accelerator.print(
+                f"override steps. steps for {args.max_train_epochs} epochs is / 指定エポックまでのステップ数: {args.max_train_steps}"
+            )
+
+        # データセット側にも学習ステップを送信
+        train_dataset_group.set_max_train_steps(args.max_train_steps)
+
+        # lr schedulerを用意する
+        lr_scheduler = train_util.get_scheduler_fix(args, optimizer, accelerator.num_processes)
+
+        # 実験的機能：勾配も含めたfp16学習を行う　モデル全体をfp16にする
+        if args.full_fp16:
+            assert (
+                args.mixed_precision == "fp16"
+            ), "full_fp16 requires mixed precision='fp16' / full_fp16を使う場合はmixed_precision='fp16'を指定してください。"
+            accelerator.print("enable full fp16 training.")
+            network.to(weight_dtype)
+
+        # acceleratorがなんかよろしくやってくれるらしい
+        # TODO めちゃくちゃ冗長なのでコードを整理する
+        if train_unet and train_text_encoder:
+            if len(text_encoders) > 1:
+                unet, t_enc1, t_enc2, network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+                    unet, text_encoders[0], text_encoders[1], network, optimizer, train_dataloader, lr_scheduler
+                )
+                text_encoder = text_encoders = [t_enc1, t_enc2]
+                del t_enc1, t_enc2
+            else:
+                unet, text_encoder, network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+                    unet, text_encoder, network, optimizer, train_dataloader, lr_scheduler
+                )
+                text_encoders = [text_encoder]
+        elif train_unet:
+            unet, network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+                unet, network, optimizer, train_dataloader, lr_scheduler
+            )
+        elif train_text_encoder:
+            if len(text_encoders) > 1:
+                t_enc1, t_enc2, network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+                    text_encoders[0], text_encoders[1], network, optimizer, train_dataloader, lr_scheduler
+                )
+                text_encoder = text_encoders = [t_enc1, t_enc2]
+                del t_enc1, t_enc2
+            else:
+                text_encoder, network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+                    text_encoder, network, optimizer, train_dataloader, lr_scheduler
+                )
+                text_encoders = [text_encoder]
+        else:
+            network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+                network, optimizer, train_dataloader, lr_scheduler
+            )
+
+        # transform DDP after prepare (train_network here only)
+        text_encoders = train_util.transform_models_if_DDP(text_encoders)
+        unet, network = train_util.transform_models_if_DDP([unet, network])
+
+        unet.requires_grad_(False)
+        unet.to(accelerator.device, dtype=weight_dtype)
+        for t_enc in text_encoders:
+            t_enc.requires_grad_(False)
+
+        if args.gradient_checkpointing:
+            # according to TI example in Diffusers, train is required
+            unet.train()
+            for t_enc in text_encoders:
+                t_enc.train()
+
+                # set top parameter requires_grad = True for gradient checkpointing works
+                t_enc.text_model.embeddings.requires_grad_(True)
+        else:
+            unet.eval()
+            for t_enc in text_encoders:
+                t_enc.eval()
+
+        del t_enc
+
+        network.prepare_grad_etc(text_encoder, unet)
+
+        if not cache_latents:  # キャッシュしない場合はVAEを使うのでVAEを準備する
+            vae.requires_grad_(False)
+            vae.eval()
+            vae.to(accelerator.device, dtype=vae_dtype)
+
+        # 実験的機能：勾配も含めたfp16学習を行う　PyTorchにパッチを当ててfp16でのgrad scaleを有効にする
+        if args.full_fp16:
+            train_util.patch_accelerator_for_fp16_training(accelerator)
+
+        # resumeする
+        train_util.resume_from_local_or_hf_if_specified(accelerator, args)
+
+        # epoch数を計算する
+        num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+        num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+        if (args.save_n_epoch_ratio is not None) and (args.save_n_epoch_ratio > 0):
+            args.save_every_n_epochs = math.floor(num_train_epochs / args.save_n_epoch_ratio) or 1
+
+        # 学習する
+        # TODO: find a way to handle total batch size when there are multiple datasets
+        total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
+
+        accelerator.print("running training / 学習開始")
+        accelerator.print(f"  num train images * repeats / 学習画像の数×繰り返し回数: {train_dataset_group.num_train_images}")
+        accelerator.print(f"  num reg images / 正則化画像の数: {train_dataset_group.num_reg_images}")
+        accelerator.print(f"  num batches per epoch / 1epochのバッチ数: {len(train_dataloader)}")
+        accelerator.print(f"  num epochs / epoch数: {num_train_epochs}")
+        accelerator.print(
+            f"  batch size per device / バッチサイズ: {', '.join([str(d.batch_size) for d in train_dataset_group.datasets])}"
+        )
+        # accelerator.print(f"  total train batch size (with parallel & distributed & accumulation) / 総バッチサイズ（並列学習、勾配合計含む）: {total_batch_size}")
+        accelerator.print(f"  gradient accumulation steps / 勾配を合計するステップ数 = {args.gradient_accumulation_steps}")
+        accelerator.print(f"  total optimization steps / 学習ステップ数: {args.max_train_steps}")
+
+        # TODO refactor metadata creation and move to util
+        metadata = {
+            "ss_session_id": session_id,  # random integer indicating which group of epochs the model came from
+            "ss_training_started_at": training_started_at,  # unix timestamp
+            "ss_output_name": args.output_name,
+            "ss_learning_rate": args.learning_rate,
+            "ss_text_encoder_lr": args.text_encoder_lr,
+            "ss_unet_lr": args.unet_lr,
+            "ss_num_train_images": train_dataset_group.num_train_images,
+            "ss_num_reg_images": train_dataset_group.num_reg_images,
+            "ss_num_batches_per_epoch": len(train_dataloader),
+            "ss_num_epochs": num_train_epochs,
+            "ss_gradient_checkpointing": args.gradient_checkpointing,
+            "ss_gradient_accumulation_steps": args.gradient_accumulation_steps,
+            "ss_max_train_steps": args.max_train_steps,
+            "ss_lr_warmup_steps": args.lr_warmup_steps,
+            "ss_lr_scheduler": args.lr_scheduler,
+            "ss_network_module": args.network_module,
+            "ss_network_dim": args.network_dim,  # None means default because another network than LoRA may have another default dim
+            "ss_network_alpha": args.network_alpha,  # some networks may not have alpha
+            "ss_network_dropout": args.network_dropout,  # some networks may not have dropout
+            "ss_mixed_precision": args.mixed_precision,
+            "ss_full_fp16": bool(args.full_fp16),
+            "ss_v2": bool(args.v2),
+            "ss_base_model_version": model_version,
+            "ss_clip_skip": args.clip_skip,
+            "ss_max_token_length": args.max_token_length,
+            "ss_cache_latents": bool(args.cache_latents),
+            "ss_seed": args.seed,
+            "ss_lowram": args.lowram,
+            "ss_noise_offset": args.noise_offset,
+            "ss_multires_noise_iterations": args.multires_noise_iterations,
+            "ss_multires_noise_discount": args.multires_noise_discount,
+            "ss_adaptive_noise_scale": args.adaptive_noise_scale,
+            "ss_training_comment": args.training_comment,  # will not be updated after training
+            "ss_sd_scripts_commit_hash": train_util.get_git_revision_hash(),
+            "ss_optimizer": optimizer_name + (f"({optimizer_args})" if len(optimizer_args) > 0 else ""),
+            "ss_max_grad_norm": args.max_grad_norm,
+            "ss_caption_dropout_rate": args.caption_dropout_rate,
+            "ss_caption_dropout_every_n_epochs": args.caption_dropout_every_n_epochs,
+            "ss_caption_tag_dropout_rate": args.caption_tag_dropout_rate,
+            "ss_face_crop_aug_range": args.face_crop_aug_range,
+            "ss_prior_loss_weight": args.prior_loss_weight,
+            "ss_min_snr_gamma": args.min_snr_gamma,
+            "ss_scale_weight_norms": args.scale_weight_norms,
+        }
+
+        if use_user_config:
+            # save metadata of multiple datasets
+            # NOTE: pack "ss_datasets" value as json one time
+            #   or should also pack nested collections as json?
+            datasets_metadata = []
+            tag_frequency = {}  # merge tag frequency for metadata editor
+            dataset_dirs_info = {}  # merge subset dirs for metadata editor
+
+            for dataset in train_dataset_group.datasets:
+                is_dreambooth_dataset = isinstance(dataset, DreamBoothDataset)
+                dataset_metadata = {
+                    "is_dreambooth": is_dreambooth_dataset,
+                    "batch_size_per_device": dataset.batch_size,
+                    "num_train_images": dataset.num_train_images,  # includes repeating
+                    "num_reg_images": dataset.num_reg_images,
+                    "resolution": (dataset.width, dataset.height),
+                    "enable_bucket": bool(dataset.enable_bucket),
+                    "min_bucket_reso": dataset.min_bucket_reso,
+                    "max_bucket_reso": dataset.max_bucket_reso,
+                    "tag_frequency": dataset.tag_frequency,
+                    "bucket_info": dataset.bucket_info,
+                }
+
+                subsets_metadata = []
+                for subset in dataset.subsets:
+                    subset_metadata = {
+                        "img_count": subset.img_count,
+                        "num_repeats": subset.num_repeats,
+                        "color_aug": bool(subset.color_aug),
+                        "flip_aug": bool(subset.flip_aug),
+                        "random_crop": bool(subset.random_crop),
+                        "shuffle_caption": bool(subset.shuffle_caption),
+                        "keep_tokens": subset.keep_tokens,
+                    }
+
+                    image_dir_or_metadata_file = None
+                    if subset.image_dir:
+                        image_dir = os.path.basename(subset.image_dir)
+                        subset_metadata["image_dir"] = image_dir
+                        image_dir_or_metadata_file = image_dir
+
+                    if is_dreambooth_dataset:
+                        subset_metadata["class_tokens"] = subset.class_tokens
+                        subset_metadata["is_reg"] = subset.is_reg
+                        if subset.is_reg:
+                            image_dir_or_metadata_file = None  # not merging reg dataset
+                    else:
+                        metadata_file = os.path.basename(subset.metadata_file)
+                        subset_metadata["metadata_file"] = metadata_file
+                        image_dir_or_metadata_file = metadata_file  # may overwrite
+
+                    subsets_metadata.append(subset_metadata)
+
+                    # merge dataset dir: not reg subset only
+                    # TODO update additional-network extension to show detailed dataset config from metadata
+                    if image_dir_or_metadata_file is not None:
+                        # datasets may have a certain dir multiple times
+                        v = image_dir_or_metadata_file
+                        i = 2
+                        while v in dataset_dirs_info:
+                            v = image_dir_or_metadata_file + f" ({i})"
+                            i += 1
+                        image_dir_or_metadata_file = v
+
+                        dataset_dirs_info[image_dir_or_metadata_file] = {
+                            "n_repeats": subset.num_repeats,
+                            "img_count": subset.img_count,
+                        }
+
+                dataset_metadata["subsets"] = subsets_metadata
+                datasets_metadata.append(dataset_metadata)
+
+                # merge tag frequency:
+                for ds_dir_name, ds_freq_for_dir in dataset.tag_frequency.items():
+                    # あるディレクトリが複数のdatasetで使用されている場合、一度だけ数える
+                    # もともと繰り返し回数を指定しているので、キャプション内でのタグの出現回数と、それが学習で何度使われるかは一致しない
+                    # なので、ここで複数datasetの回数を合算してもあまり意味はない
+                    if ds_dir_name in tag_frequency:
+                        continue
+                    tag_frequency[ds_dir_name] = ds_freq_for_dir
+
+            metadata["ss_datasets"] = json.dumps(datasets_metadata)
+            metadata["ss_tag_frequency"] = json.dumps(tag_frequency)
+            metadata["ss_dataset_dirs"] = json.dumps(dataset_dirs_info)
+        else:
+            # conserving backward compatibility when using train_dataset_dir and reg_dataset_dir
+            assert (
+                len(train_dataset_group.datasets) == 1
+            ), f"There should be a single dataset but {len(train_dataset_group.datasets)} found. This seems to be a bug. / データセットは1個だけ存在するはずですが、実際には{len(train_dataset_group.datasets)}個でした。プログラムのバグかもしれません。"
+
+            dataset = train_dataset_group.datasets[0]
+
+            dataset_dirs_info = {}
+            reg_dataset_dirs_info = {}
+            if use_dreambooth_method:
+                for subset in dataset.subsets:
+                    info = reg_dataset_dirs_info if subset.is_reg else dataset_dirs_info
+                    info[os.path.basename(subset.image_dir)] = {"n_repeats": subset.num_repeats, "img_count": subset.img_count}
+            else:
+                for subset in dataset.subsets:
+                    dataset_dirs_info[os.path.basename(subset.metadata_file)] = {
+                        "n_repeats": subset.num_repeats,
+                        "img_count": subset.img_count,
+                    }
+
+            metadata.update(
+                {
+                    "ss_batch_size_per_device": args.train_batch_size,
+                    "ss_total_batch_size": total_batch_size,
+                    "ss_resolution": args.resolution,
+                    "ss_color_aug": bool(args.color_aug),
+                    "ss_flip_aug": bool(args.flip_aug),
+                    "ss_random_crop": bool(args.random_crop),
+                    "ss_shuffle_caption": bool(args.shuffle_caption),
+                    "ss_enable_bucket": bool(dataset.enable_bucket),
+                    "ss_bucket_no_upscale": bool(dataset.bucket_no_upscale),
+                    "ss_min_bucket_reso": dataset.min_bucket_reso,
+                    "ss_max_bucket_reso": dataset.max_bucket_reso,
+                    "ss_keep_tokens": args.keep_tokens,
+                    "ss_dataset_dirs": json.dumps(dataset_dirs_info),
+                    "ss_reg_dataset_dirs": json.dumps(reg_dataset_dirs_info),
+                    "ss_tag_frequency": json.dumps(dataset.tag_frequency),
+                    "ss_bucket_info": json.dumps(dataset.bucket_info),
+                }
+            )
+
+        # add extra args
+        if args.network_args:
+            metadata["ss_network_args"] = json.dumps(net_kwargs)
+
+        # model name and hash
+        if args.pretrained_model_name_or_path is not None:
+            sd_model_name = args.pretrained_model_name_or_path
+            if os.path.exists(sd_model_name):
+                metadata["ss_sd_model_hash"] = train_util.model_hash(sd_model_name)
+                metadata["ss_new_sd_model_hash"] = train_util.calculate_sha256(sd_model_name)
+                sd_model_name = os.path.basename(sd_model_name)
+            metadata["ss_sd_model_name"] = sd_model_name
+
+        if args.vae is not None:
+            vae_name = args.vae
+            if os.path.exists(vae_name):
+                metadata["ss_vae_hash"] = train_util.model_hash(vae_name)
+                metadata["ss_new_vae_hash"] = train_util.calculate_sha256(vae_name)
+                vae_name = os.path.basename(vae_name)
+            metadata["ss_vae_name"] = vae_name
+
+        metadata = {k: str(v) for k, v in metadata.items()}
+
+        # make minimum metadata for filtering
+        minimum_keys = [
+            "ss_v2",
+            "ss_base_model_version",
+            "ss_network_module",
+            "ss_network_dim",
+            "ss_network_alpha",
+            "ss_network_args",
+        ]
+        minimum_metadata = {}
+        for key in minimum_keys:
+            if key in metadata:
+                minimum_metadata[key] = metadata[key]
+
+        progress_bar = tqdm(range(args.max_train_steps), smoothing=0, disable=not accelerator.is_local_main_process, desc="steps")
+        global_step = 0
+
+        noise_scheduler = DDPMScheduler(
+            beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, clip_sample=False
+        )
+        prepare_scheduler_for_custom_training(noise_scheduler, accelerator.device)
+
+        if accelerator.is_main_process:
+            accelerator.init_trackers("network_train" if args.log_tracker_name is None else args.log_tracker_name)
+
+        loss_list = []
+        loss_total = 0.0
+        del train_dataset_group
+
+        # callback for step start
+        if hasattr(network, "on_step_start"):
+            on_step_start = network.on_step_start
+        else:
+            on_step_start = lambda *args, **kwargs: None
+
+        # function for saving/removing
+        def save_model(ckpt_name, unwrapped_nw, steps, epoch_no, force_sync_upload=False):
+            os.makedirs(args.output_dir, exist_ok=True)
+            ckpt_file = os.path.join(args.output_dir, ckpt_name)
+
+            accelerator.print(f"\nsaving checkpoint: {ckpt_file}")
+            metadata["ss_training_finished_at"] = str(time.time())
+            metadata["ss_steps"] = str(steps)
+            metadata["ss_epoch"] = str(epoch_no)
+
+            unwrapped_nw.save_weights(ckpt_file, save_dtype, minimum_metadata if args.no_metadata else metadata)
+            if args.huggingface_repo_id is not None:
+                huggingface_util.upload(args, ckpt_file, "/" + ckpt_name, force_sync_upload=force_sync_upload)
+
+        def remove_model(old_ckpt_name):
+            old_ckpt_file = os.path.join(args.output_dir, old_ckpt_name)
+            if os.path.exists(old_ckpt_file):
+                accelerator.print(f"removing old checkpoint: {old_ckpt_file}")
+                os.remove(old_ckpt_file)
+
+        # training loop
+        for epoch in range(num_train_epochs):
+            accelerator.print(f"\nepoch {epoch+1}/{num_train_epochs}")
+            current_epoch.value = epoch + 1
+
+            metadata["ss_epoch"] = str(epoch + 1)
+
+            network.on_epoch_start(text_encoder, unet)
+
+            for step, batch in enumerate(train_dataloader):
+                current_step.value = global_step
+                with accelerator.accumulate(network):
+                    on_step_start(text_encoder, unet)
+
+                    with torch.no_grad():
+                        if "latents" in batch and batch["latents"] is not None:
+                            latents = batch["latents"].to(accelerator.device)
+                        else:
+                            # latentに変換
+                            latents = vae.encode(batch["images"].to(dtype=vae_dtype)).latent_dist.sample()
+
+                            # NaNが含まれていれば警告を表示し0に置き換える
+                            if torch.any(torch.isnan(latents)):
+                                accelerator.print("NaN found in latents, replacing with zeros")
+                                latents = torch.where(torch.isnan(latents), torch.zeros_like(latents), latents)
+                        latents = latents * self.vae_scale_factor
+                    b_size = latents.shape[0]
+
+                    with torch.set_grad_enabled(train_text_encoder):
+                        # Get the text embedding for conditioning
+                        if args.weighted_captions:
+                            text_encoder_conds = get_weighted_text_embeddings(
+                                tokenizer,
+                                text_encoder,
+                                batch["captions"],
+                                accelerator.device,
+                                args.max_token_length // 75 if args.max_token_length else 1,
+                                clip_skip=args.clip_skip,
+                            )
+                        else:
+                            text_encoder_conds = self.get_text_cond(
+                                args, accelerator, batch, tokenizers, text_encoders, weight_dtype
+                            )
+
+                    # Sample noise, sample a random timestep for each image, and add noise to the latents,
+                    # with noise offset and/or multires noise if specified
+                    noise, noisy_latents, timesteps = train_util.get_noise_noisy_latents_and_timesteps(
+                        args, noise_scheduler, latents
+                    )
+
+                    # Predict the noise residual
+                    with accelerator.autocast():
+                        noise_pred = self.call_unet(
+                            args, accelerator, unet, noisy_latents, timesteps, text_encoder_conds, batch, weight_dtype
+                        )
+
+                    if args.v_parameterization:
+                        # v-parameterization training
+                        target = noise_scheduler.get_velocity(latents, noise, timesteps)
+                    else:
+                        target = noise
+
+                    loss = torch.nn.functional.mse_loss(noise_pred.float(), target.float(), reduction="none")
+                    loss = loss.mean([1, 2, 3])
+
+                    loss_weights = batch["loss_weights"]  # 各sampleごとのweight
+                    loss = loss * loss_weights
+
+                    if args.min_snr_gamma:
+                        loss = apply_snr_weight(loss, timesteps, noise_scheduler, args.min_snr_gamma)
+                    if args.scale_v_pred_loss_like_noise_pred:
+                        loss = scale_v_prediction_loss_like_noise_prediction(loss, timesteps, noise_scheduler)
+
+                    loss = loss.mean()  # 平均なのでbatch_sizeで割る必要なし
+
+                    accelerator.backward(loss)
+                    if accelerator.sync_gradients and args.max_grad_norm != 0.0:
+                        params_to_clip = network.get_trainable_params()
+                        accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
+
+                    optimizer.step()
+                    lr_scheduler.step()
+                    optimizer.zero_grad(set_to_none=True)
+
+                if args.scale_weight_norms:
+                    keys_scaled, mean_norm, maximum_norm = network.apply_max_norm_regularization(
+                        args.scale_weight_norms, accelerator.device
+                    )
+                    max_mean_logs = {"Keys Scaled": keys_scaled, "Average key norm": mean_norm}
+                else:
+                    keys_scaled, mean_norm, maximum_norm = None, None, None
+
+                # Checks if the accelerator has performed an optimization step behind the scenes
+                if accelerator.sync_gradients:
+                    progress_bar.update(1)
+                    global_step += 1
+
+                    self.sample_images(accelerator, args, None, global_step, accelerator.device, vae, tokenizer, text_encoder, unet)
+
+                    # 指定ステップごとにモデルを保存
+                    if args.save_every_n_steps is not None and global_step % args.save_every_n_steps == 0:
+                        accelerator.wait_for_everyone()
+                        if accelerator.is_main_process:
+                            ckpt_name = train_util.get_step_ckpt_name(args, "." + args.save_model_as, global_step)
+                            save_model(ckpt_name, accelerator.unwrap_model(network), global_step, epoch)
+
+                            if args.save_state:
+                                train_util.save_and_remove_state_stepwise(args, accelerator, global_step)
+
+                            remove_step_no = train_util.get_remove_step_no(args, global_step)
+                            if remove_step_no is not None:
+                                remove_ckpt_name = train_util.get_step_ckpt_name(args, "." + args.save_model_as, remove_step_no)
+                                remove_model(remove_ckpt_name)
+
+                current_loss = loss.detach().item()
+                if epoch == 0:
+                    loss_list.append(current_loss)
+                else:
+                    loss_total -= loss_list[step]
+                    loss_list[step] = current_loss
+                loss_total += current_loss
+                avr_loss = loss_total / len(loss_list)
+                logs = {"loss": avr_loss}  # , "lr": lr_scheduler.get_last_lr()[0]}
+                progress_bar.set_postfix(**logs)
+
+                if args.scale_weight_norms:
+                    progress_bar.set_postfix(**{**max_mean_logs, **logs})
+
+                if args.logging_dir is not None:
+                    logs = self.generate_step_logs(args, current_loss, avr_loss, lr_scheduler, keys_scaled, mean_norm, maximum_norm)
+                    accelerator.log(logs, step=global_step)
+
+                if global_step >= args.max_train_steps:
+                    break
+
+            if args.logging_dir is not None:
+                logs = {"loss/epoch": loss_total / len(loss_list)}
+                accelerator.log(logs, step=epoch + 1)
+
+            accelerator.wait_for_everyone()
+
+            # 指定エポックごとにモデルを保存
+            if args.save_every_n_epochs is not None:
+                saving = (epoch + 1) % args.save_every_n_epochs == 0 and (epoch + 1) < num_train_epochs
+                if is_main_process and saving:
+                    ckpt_name = train_util.get_epoch_ckpt_name(args, "." + args.save_model_as, epoch + 1)
+                    save_model(ckpt_name, accelerator.unwrap_model(network), global_step, epoch + 1)
+
+                    remove_epoch_no = train_util.get_remove_epoch_no(args, epoch + 1)
+                    if remove_epoch_no is not None:
+                        remove_ckpt_name = train_util.get_epoch_ckpt_name(args, "." + args.save_model_as, remove_epoch_no)
+                        remove_model(remove_ckpt_name)
+
+                    if args.save_state:
+                        train_util.save_and_remove_state_on_epoch_end(args, accelerator, epoch + 1)
+
+            self.sample_images(accelerator, args, epoch + 1, global_step, accelerator.device, vae, tokenizer, text_encoder, unet)
+
+            # end of epoch
+
+        # metadata["ss_epoch"] = str(num_train_epochs)
+        metadata["ss_training_finished_at"] = str(time.time())
+
+        if is_main_process:
+            network = accelerator.unwrap_model(network)
+
+        accelerator.end_training()
+
+        if is_main_process and args.save_state:
+            train_util.save_state_on_train_end(args, accelerator)
+
+        if is_main_process:
+            ckpt_name = train_util.get_last_ckpt_name(args, "." + args.save_model_as)
+            save_model(ckpt_name, network, global_step, num_train_epochs, force_sync_upload=True)
+
+            print("model saved.")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    train_util.add_sd_models_arguments(parser)
+    train_util.add_dataset_arguments(parser, True, True, True)
+    train_util.add_training_arguments(parser, True)
+    train_util.add_optimizer_arguments(parser)
+    config_util.add_config_arguments(parser)
+    custom_train_functions.add_custom_train_arguments(parser)
+
+    parser.add_argument("--no_metadata", action="store_true", help="do not save metadata in output model / メタデータを出力先モデルに保存しない")
+    parser.add_argument(
+        "--save_model_as",
+        type=str,
+        default="safetensors",
+        choices=[None, "ckpt", "pt", "safetensors"],
+        help="format to save the model (default is .safetensors) / モデル保存時の形式（デフォルトはsafetensors）",
+    )
+
+    parser.add_argument("--unet_lr", type=float, default=None, help="learning rate for U-Net / U-Netの学習率")
+    parser.add_argument("--text_encoder_lr", type=float, default=None, help="learning rate for Text Encoder / Text Encoderの学習率")
+
+    parser.add_argument("--network_weights", type=str, default=None, help="pretrained weights for network / 学習するネットワークの初期重み")
+    parser.add_argument("--network_module", type=str, default=None, help="network module to train / 学習対象のネットワークのモジュール")
+    parser.add_argument(
+        "--network_dim", type=int, default=None, help="network dimensions (depends on each network) / モジュールの次元数（ネットワークにより定義は異なります）"
+    )
+    parser.add_argument(
+        "--network_alpha",
+        type=float,
+        default=1,
+        help="alpha for LoRA weight scaling, default 1 (same as network_dim for same behavior as old version) / LoRaの重み調整のalpha値、デフォルト1（旧バージョンと同じ動作をするにはnetwork_dimと同じ値を指定）",
+    )
+    parser.add_argument(
+        "--network_dropout",
+        type=float,
+        default=None,
+        help="Drops neurons out of training every step (0 or None is default behavior (no dropout), 1 would drop all neurons) / 訓練時に毎ステップでニューロンをdropする（0またはNoneはdropoutなし、1は全ニューロンをdropout）",
+    )
+    parser.add_argument(
+        "--network_args", type=str, default=None, nargs="*", help="additional argmuments for network (key=value) / ネットワークへの追加の引数"
+    )
+    parser.add_argument("--network_train_unet_only", action="store_true", help="only training U-Net part / U-Net関連部分のみ学習する")
+    parser.add_argument(
+        "--network_train_text_encoder_only", action="store_true", help="only training Text Encoder part / Text Encoder関連部分のみ学習する"
+    )
+    parser.add_argument(
+        "--training_comment", type=str, default=None, help="arbitrary comment string stored in metadata / メタデータに記録する任意のコメント文字列"
+    )
+    parser.add_argument(
+        "--dim_from_weights",
+        action="store_true",
+        help="automatically determine dim (rank) from network_weights / dim (rank)をnetwork_weightsで指定した重みから自動で決定する",
+    )
+    parser.add_argument(
+        "--scale_weight_norms",
+        type=float,
+        default=None,
+        help="Scale the weight of each key pair to help prevent overtraing via exploding gradients. (1 is a good starting point) / 重みの値をスケーリングして勾配爆発を防ぐ（1が初期値としては適当）",
+    )
+    parser.add_argument(
+        "--base_weights",
+        type=str,
+        default=None,
+        nargs="*",
+        help="network weights to merge into the model before training / 学習前にあらかじめモデルにマージするnetworkの重みファイル",
+    )
+    parser.add_argument(
+        "--base_weights_multiplier",
+        type=float,
+        default=None,
+        nargs="*",
+        help="multiplier for network weights to merge into the model before training / 学習前にあらかじめモデルにマージするnetworkの重みの倍率",
+    )
+    parser.add_argument(
+        "--no_half_vae",
+        action="store_true",
+        help="do not use fp16/bf16 VAE in mixed precision (use float VAE) / mixed precisionでも fp16/bf16 VAEを使わずfloat VAEを使う",
+    )
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    args = train_util.read_config_from_file(args, parser)
+
+    trainer = NetworkTrainer()
+    trainer.train(args)
diff --git a/train_network_README.md b/train_network_README.md
new file mode 100644
index 0000000000000000000000000000000000000000..b0363a68b66f4fb8ddd7efc5ccaecc730e1666e9
--- /dev/null
+++ b/train_network_README.md
@@ -0,0 +1,189 @@
+## About learning LoRA
+
+[LoRA: Low-Rank Adaptation of Large Language Models](https://arxiv.org/abs/2106.09685) (arxiv), [LoRA](https://github.com/microsoft/LoRA) (github) to Stable Applied to Diffusion.
+
+[cloneofsimo's repository](https://github.com/cloneofsimo/lora) was a great reference. Thank you very much.
+
+8GB VRAM seems to work just fine.
+
+## A Note about Trained Models
+
+Cloneofsimo's repository and d8ahazard's [Drebooth Extension for Stable-Diffusion-WebUI](https://github.com/d8ahazard/sd_drebooth_extension) are currently incompatible. Because we are doing some enhancements (see below).
+
+When generating images with WebUI, etc., merge the learned LoRA model with the learning source Stable Diffusion model in advance with the script in this repository, or click here [Extention for WebUI] (https://github .com/kohya-ss/sd-webui-additional-networks).
+
+## Learning method
+
+Use train_network.py.
+
+You can learn both the DreamBooth method (using identifiers (sks, etc.) and classes, optionally regularized images) and the fine tuning method using captions.
+
+Both methods can be learned in much the same way as existing scripts. We will discuss the differences later.
+
+### Using the DreamBooth Method
+
+Please refer to [DreamBooth guide](./train_db_README-en.md) and prepare the data.
+
+Specify train_network.py instead of train_db.py when training.
+
+Almost all options are available (except Stable Diffusion model save related), but stop_text_encoder_training is not supported.
+
+### When to use captions
+
+Please refer to [fine-tuning guide](./fine_tune_README_en.md) and perform each step.
+
+Specify train_network.py instead of fine_tune.py when training. Almost all options (except for model saving) can be used as is.
+
+In addition, it will work even if you do not perform "Pre-obtain latents". Since the latent is acquired from the VAE when learning (or caching), the learning speed will be slower, but color_aug can be used instead.
+
+### Options for Learning LoRA
+
+In train_network.py, specify the name of the module to be trained in the --network_module option. LoRA is compatible with network.lora, so please specify it.
+
+The learning rate should be set to about 1e-4, which is higher than normal DreamBooth and fine tuning.
+
+Below is an example command line (DreamBooth technique).
+
+```
+accelerate launch --num_cpu_threads_per_process 12 train_network.py
+     --pretrained_model_name_or_path=..\models\model.ckpt
+     --train_data_dir=..\data\db\char1 --output_dir=..\lora_train1
+     --reg_data_dir=..\data\db\reg1 --prior_loss_weight=1.0
+     --resolution=448,640 --train_batch_size=1 --learning_rate=1e-4
+     --max_train_steps=400 --use_8bit_adam --xformers --mixed_precision=fp16
+     --save_every_n_epochs=1 --save_model_as=safetensors --clip_skip=2 --seed=42 --color_aug
+     --network_module=networks.lora
+```
+
+The LoRA model will be saved in the directory specified by the --output_dir option.
+
+In addition, the following options can be specified.
+
+* --network_dim
+   * Specify the number of dimensions of LoRA (such as ``--networkdim=4``). Default is 4. The greater the number, the greater the expressive power, but the memory and time required for learning also increase. In addition, it seems that it is not good to increase it blindly.
+* --network_weights
+   * Load pretrained LoRA weights before training and additionally learn from them.
+* --network_train_unet_only
+   * Valid only for LoRA modules related to U-Net. It may be better to specify it in fine-tuning study.
+* --network_train_text_encoder_only
+   * Only LoRA modules related to Text Encoder are enabled. You may be able to expect a textual inversion effect.
+* --unet_lr
+   * Specify when using a learning rate different from the normal learning rate (specified with the --learning_rate option) for the LoRA module related to U-Net.
+* --text_encoder_lr
+   * Specify when using a learning rate different from the normal learning rate (specified with the --learning_rate option) for the LoRA module associated with the Text Encoder. Some people say that it is better to set the Text Encoder to a slightly lower learning rate (such as 5e-5).
+
+When neither --network_train_unet_only nor --network_train_text_encoder_only is specified (default), both Text Encoder and U-Net LoRA modules are enabled.
+
+## About the merge script
+
+merge_lora.py allows you to merge LoRA training results into a Stable Diffusion model, or merge multiple LoRA models.
+
+### Merge LoRA model into Stable Diffusion model
+
+The model after merging can be handled in the same way as normal Stable Diffusion ckpt. For example, a command line like:
+
+```
+python networks\merge_lora.py --sd_model ..\model\model.ckpt
+     --save_to ..\lora_train1\model-char1-merged.safetensors
+     --models ..\lora_train1\last.safetensors --ratios 0.8
+```
+
+Specify the --v2 option if you want to train with a Stable Diffusion v2.x model and merge with it.
+
+Specify the Stable Diffusion model file to be merged in the --sd_model option (only .ckpt or .safetensors are supported, Diffusers is not currently supported).
+
+Specify the save destination of the model after merging in the --save_to option (.ckpt or .safetensors, automatically determined by extension).
+
+Specify the LoRA model file learned in --models. It is possible to specify more than one, in which case they will be merged in order.
+
+For --ratios, specify the application rate of each model (how much weight is reflected in the original model) with a numerical value from 0 to 1.0. For example, if it is close to overfitting, it may be better if the application rate is lowered. Specify as many as the number of models.
+
+When specifying multiple, it will be as follows.
+
+```
+python networks\merge_lora.py --sd_model ..\model\model.ckpt
+     --save_to ..\lora_train1\model-char1-merged.safetensors
+     --models ..\lora_train1\last.safetensors ..\lora_train2\last.safetensors --ratios 0.8 0.5
+```
+
+### Merge multiple LoRA models
+
+Applying multiple LoRA models one by one to the SD model and merging multiple LoRA models and then merging them into the SD model yield slightly different results in relation to the calculation order.
+
+For example, a command line like:
+
+```
+python networks\merge_lora.py
+     --save_to ..\lora_train1\model-char1-style1-merged.safetensors
+     --models ..\lora_train1\last.safetensors ..\lora_train2\last.safetensors --ratios 0.6 0.4
+```
+
+The --sd_model option does not need to be specified.
+
+Specify the save destination of the merged LoRA model in the --save_to option (.ckpt or .safetensors, automatically determined by extension).
+
+Specify the LoRA model file learned in --models. Three or more can be specified.
+
+For --ratios, specify the ratio of each model (how much weight is reflected in the original model) with a numerical value from 0 to 1.0. If you merge two models one-to-one, it will be "0.5 0.5". "1.0 1.0" would give too much weight to the sum, and the result would probably be less desirable.
+
+LoRA trained with v1 and LoRA trained with v2, and LoRA with different number of dimensions cannot be merged. U-Net only LoRA and U-Net+Text Encoder LoRA should be able to merge, but the result is unknown.
+
+
+### Other Options
+
+* precision
+   * The precision for merge calculation can be specified from float, fp16, and bf16. If omitted, it will be float to ensure accuracy. Specify fp16/bf16 if you want to reduce memory usage.
+* save_precision
+   * You can specify the precision when saving the model from float, fp16, bf16. If omitted, the precision is the same as precision.
+
+## Generate with the image generation script in this repository
+
+Add options --network_module, --network_weights, --network_dim (optional) to gen_img_diffusers.py. The meaning is the same as when learning.
+
+You can change the LoRA application rate by specifying a value between 0 and 1.0 with the --network_mul option.
+
+## Create a LoRA model from the difference between two models
+
+It was implemented with reference to [this discussion](https://github.com/cloneofsimo/lora/discussions/56). I used the formula as it is (I don't understand it well, but it seems that singular value decomposition is used for approximation).
+
+LoRA approximates the difference between two models (for example, the original model after fine tuning and the model after fine tuning).
+
+### How to run scripts
+
+Please specify as follows.
+```
+python networks\extract_lora_from_models.py --model_org base-model.ckpt
+     --model_tuned fine-tuned-model.ckpt
+     --save_to lora-weights.safetensors --dim 4
+```
+
+Specify the original Stable Diffusion model for the --model_org option. When applying the created LoRA model, this model will be specified and applied. .ckpt or .safetensors can be specified.
+
+Specify the Stable Diffusion model to extract the difference in the --model_tuned option. For example, specify a model after fine tuning or DreamBooth. .ckpt or .safetensors can be specified.
+
+Specify the save destination of the LoRA model in --save_to. Specify the number of dimensions of LoRA in --dim.
+
+A generated LoRA model can be used in the same way as a trained LoRA model.
+
+If the Text Encoder is the same for both models, LoRA will be U-Net only LoRA.
+
+### Other Options
+
+--v2
+   - Please specify when using the v2.x Stable Diffusion model.
+--device
+   - If cuda is specified as ``--device cuda``, the calculation will be performed on the GPU. Processing will be faster (because even the CPU is not that slow, it seems to be at most twice or several times faster).
+--save_precision
+   - Specify the LoRA save format from "float", "fp16", "bf16". Default is float.
+
+## Additional Information
+
+### Differences from cloneofsimo's repository
+
+As of 12/25, this repository has expanded LoRA application points to Text Encoder's MLP, U-Net's FFN, and Transformer's in/out projection, increasing its expressiveness. However, the amount of memory used increased instead, and it became the last minute of 8GB.
+
+Also, the module replacement mechanism is completely different.
+
+### About Future Expansion
+
+It is possible to support not only LoRA but also other expansions, so we plan to add them as well.
\ No newline at end of file
diff --git a/train_textual_inversion.py b/train_textual_inversion.py
new file mode 100644
index 0000000000000000000000000000000000000000..cbfd48ce7d3e2142c9b0c5d99a151661484b2d3a
--- /dev/null
+++ b/train_textual_inversion.py
@@ -0,0 +1,764 @@
+import argparse
+import gc
+import math
+import os
+from multiprocessing import Value
+
+from tqdm import tqdm
+import torch
+from accelerate.utils import set_seed
+from diffusers import DDPMScheduler
+from library import model_util
+
+import library.train_util as train_util
+import library.huggingface_util as huggingface_util
+import library.config_util as config_util
+from library.config_util import (
+    ConfigSanitizer,
+    BlueprintGenerator,
+)
+import library.custom_train_functions as custom_train_functions
+from library.custom_train_functions import (
+    apply_snr_weight,
+    prepare_scheduler_for_custom_training,
+    scale_v_prediction_loss_like_noise_prediction,
+)
+
+imagenet_templates_small = [
+    "a photo of a {}",
+    "a rendering of a {}",
+    "a cropped photo of the {}",
+    "the photo of a {}",
+    "a photo of a clean {}",
+    "a photo of a dirty {}",
+    "a dark photo of the {}",
+    "a photo of my {}",
+    "a photo of the cool {}",
+    "a close-up photo of a {}",
+    "a bright photo of the {}",
+    "a cropped photo of a {}",
+    "a photo of the {}",
+    "a good photo of the {}",
+    "a photo of one {}",
+    "a close-up photo of the {}",
+    "a rendition of the {}",
+    "a photo of the clean {}",
+    "a rendition of a {}",
+    "a photo of a nice {}",
+    "a good photo of a {}",
+    "a photo of the nice {}",
+    "a photo of the small {}",
+    "a photo of the weird {}",
+    "a photo of the large {}",
+    "a photo of a cool {}",
+    "a photo of a small {}",
+]
+
+imagenet_style_templates_small = [
+    "a painting in the style of {}",
+    "a rendering in the style of {}",
+    "a cropped painting in the style of {}",
+    "the painting in the style of {}",
+    "a clean painting in the style of {}",
+    "a dirty painting in the style of {}",
+    "a dark painting in the style of {}",
+    "a picture in the style of {}",
+    "a cool painting in the style of {}",
+    "a close-up painting in the style of {}",
+    "a bright painting in the style of {}",
+    "a cropped painting in the style of {}",
+    "a good painting in the style of {}",
+    "a close-up painting in the style of {}",
+    "a rendition in the style of {}",
+    "a nice painting in the style of {}",
+    "a small painting in the style of {}",
+    "a weird painting in the style of {}",
+    "a large painting in the style of {}",
+]
+
+
+class TextualInversionTrainer:
+    def __init__(self):
+        self.vae_scale_factor = 0.18215
+
+    def assert_extra_args(self, args, train_dataset_group):
+        pass
+
+    def load_target_model(self, args, weight_dtype, accelerator):
+        text_encoder, vae, unet, _ = train_util.load_target_model(args, weight_dtype, accelerator)
+        return model_util.get_model_version_str_for_sd1_sd2(args.v2, args.v_parameterization), text_encoder, vae, unet
+
+    def load_tokenizer(self, args):
+        tokenizer = train_util.load_tokenizer(args)
+        return tokenizer
+
+    def assert_token_string(self, token_string, tokenizers):
+        pass
+
+    def get_text_cond(self, args, accelerator, batch, tokenizers, text_encoders, weight_dtype):
+        with torch.enable_grad():
+            input_ids = batch["input_ids"].to(accelerator.device)
+            encoder_hidden_states = train_util.get_hidden_states(args, input_ids, tokenizers[0], text_encoders[0], None)
+            return encoder_hidden_states
+
+    def call_unet(self, args, accelerator, unet, noisy_latents, timesteps, text_conds, batch, weight_dtype):
+        noise_pred = unet(noisy_latents, timesteps, text_conds).sample
+        return noise_pred
+
+    def sample_images(self, accelerator, args, epoch, global_step, device, vae, tokenizer, text_encoder, unet, prompt_replacement):
+        train_util.sample_images(
+            accelerator, args, epoch, global_step, device, vae, tokenizer, text_encoder, unet, prompt_replacement
+        )
+
+    def save_weights(self, file, updated_embs, save_dtype):
+        state_dict = {"emb_params": updated_embs[0]}
+
+        if save_dtype is not None:
+            for key in list(state_dict.keys()):
+                v = state_dict[key]
+                v = v.detach().clone().to("cpu").to(save_dtype)
+                state_dict[key] = v
+
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import save_file
+
+            save_file(state_dict, file)
+        else:
+            torch.save(state_dict, file)  # can be loaded in Web UI
+
+    def load_weights(self, file):
+        if os.path.splitext(file)[1] == ".safetensors":
+            from safetensors.torch import load_file
+
+            data = load_file(file)
+        else:
+            # compatible to Web UI's file format
+            data = torch.load(file, map_location="cpu")
+            if type(data) != dict:
+                raise ValueError(f"weight file is not dict / 重みファイルがdict形式ではありません: {file}")
+
+            if "string_to_param" in data:  # textual inversion embeddings
+                data = data["string_to_param"]
+                if hasattr(data, "_parameters"):  # support old PyTorch?
+                    data = getattr(data, "_parameters")
+
+        emb = next(iter(data.values()))
+        if type(emb) != torch.Tensor:
+            raise ValueError(f"weight file does not contains Tensor / 重みファイルのデータがTensorではありません: {file}")
+
+        if len(emb.size()) == 1:
+            emb = emb.unsqueeze(0)
+
+        return [emb]
+
+    def train(self, args):
+        if args.output_name is None:
+            args.output_name = args.token_string
+        use_template = args.use_object_template or args.use_style_template
+
+        train_util.verify_training_args(args)
+        train_util.prepare_dataset_args(args, True)
+
+        cache_latents = args.cache_latents
+
+        if args.seed is not None:
+            set_seed(args.seed)
+
+        tokenizer_or_list = self.load_tokenizer(args)  # list of tokenizer or tokenizer
+        tokenizers = tokenizer_or_list if isinstance(tokenizer_or_list, list) else [tokenizer_or_list]
+
+        # acceleratorを準備する
+        print("prepare accelerator")
+        accelerator = train_util.prepare_accelerator(args)
+
+        # mixed precisionに対応した型を用意しておき適宜castする
+        weight_dtype, save_dtype = train_util.prepare_dtype(args)
+        vae_dtype = torch.float32 if args.no_half_vae else weight_dtype
+
+        # モデルを読み込む
+        model_version, text_encoder_or_list, vae, unet = self.load_target_model(args, weight_dtype, accelerator)
+        text_encoders = [text_encoder_or_list] if not isinstance(text_encoder_or_list, list) else text_encoder_or_list
+
+        if len(text_encoders) > 1 and args.gradient_accumulation_steps > 1:
+            accelerator.print(
+                "accelerate doesn't seem to support gradient_accumulation_steps for multiple models (text encoders) / "
+                + "accelerateでは複数のモデル（テキストエンコーダー）のgradient_accumulation_stepsはサポートされていないようです"
+            )
+
+        # Convert the init_word to token_id
+        init_token_ids_list = []
+        if args.init_word is not None:
+            for i, tokenizer in enumerate(tokenizers):
+                init_token_ids = tokenizer.encode(args.init_word, add_special_tokens=False)
+                if len(init_token_ids) > 1 and len(init_token_ids) != args.num_vectors_per_token:
+                    accelerator.print(
+                        f"token length for init words is not same to num_vectors_per_token, init words is repeated or truncated / "
+                        + f"初期化単語のトークン長がnum_vectors_per_tokenと合わないため、繰り返しまたは切り捨てが発生します:  tokenizer {i+1}, length {len(init_token_ids)}"
+                    )
+                init_token_ids_list.append(init_token_ids)
+        else:
+            init_token_ids_list = [None] * len(tokenizers)
+
+        # tokenizerに新しい単語を追加する。追加する単語の数はnum_vectors_per_token
+        # add new word to tokenizer, count is num_vectors_per_token
+
+        # token_stringが hoge の場合、"hoge", "hogea", "hogeb", ... が追加される
+        # 当初は "hoge", "hoge1", "hoge2", ... としていたが、open clipのtokenizerは数字を含む単語を分割してしまうため(;^ω^)、a, b, ... とした
+
+        # if token_string is hoge, "hoge", "hogea", "hogeb", ... are added
+        # originally, "hoge", "hoge1", "hoge2", ... were used, but open clip's tokenizer splits words including numbers (;^ω^), so a, b, ... are used
+
+        self.assert_token_string(args.token_string, tokenizers)
+
+        token_strings = [args.token_string] + [
+            f"{args.token_string}{chr(ord('a') + i)}" for i in range(args.num_vectors_per_token - 1)
+        ]
+        token_ids_list = []
+        token_embeds_list = []
+        for i, (tokenizer, text_encoder, init_token_ids) in enumerate(zip(tokenizers, text_encoders, init_token_ids_list)):
+            num_added_tokens = tokenizer.add_tokens(token_strings)
+            assert (
+                num_added_tokens == args.num_vectors_per_token
+            ), f"tokenizer has same word to token string. please use another one / 指定したargs.token_stringは既に存在します。別の単語を使ってください: tokenizer {i+1}, {args.token_string}"
+
+            token_ids = tokenizer.convert_tokens_to_ids(token_strings)
+            accelerator.print(f"tokens are added for tokenizer {i+1}: {token_ids}")
+            assert (
+                min(token_ids) == token_ids[0] and token_ids[-1] == token_ids[0] + len(token_ids) - 1
+            ), f"token ids is not ordered : tokenizer {i+1}, {token_ids}"
+            assert (
+                len(tokenizer) - 1 == token_ids[-1]
+            ), f"token ids is not end of tokenize: tokenizer {i+1}, {token_ids}, {len(tokenizer)}"
+            token_ids_list.append(token_ids)
+
+            # Resize the token embeddings as we are adding new special tokens to the tokenizer
+            text_encoder.resize_token_embeddings(len(tokenizer))
+
+            # Initialise the newly added placeholder token with the embeddings of the initializer token
+            token_embeds = text_encoder.get_input_embeddings().weight.data
+            if init_token_ids is not None:
+                for i, token_id in enumerate(token_ids):
+                    token_embeds[token_id] = token_embeds[init_token_ids[i % len(init_token_ids)]]
+                    # accelerator.print(token_id, token_embeds[token_id].mean(), token_embeds[token_id].min())
+            token_embeds_list.append(token_embeds)
+
+        # load weights
+        if args.weights is not None:
+            embeddings_list = self.load_weights(args.weights)
+            assert len(token_ids) == len(
+                embeddings_list[0]
+            ), f"num_vectors_per_token is mismatch for weights / 指定した重みとnum_vectors_per_tokenの値が異なります: {len(embeddings)}"
+            # accelerator.print(token_ids, embeddings.size())
+            for token_ids, embeddings, token_embeds in zip(token_ids_list, embeddings_list, token_embeds_list):
+                for token_id, embedding in zip(token_ids, embeddings):
+                    token_embeds[token_id] = embedding
+                    # accelerator.print(token_id, token_embeds[token_id].mean(), token_embeds[token_id].min())
+            accelerator.print(f"weighs loaded")
+
+        accelerator.print(f"create embeddings for {args.num_vectors_per_token} tokens, for {args.token_string}")
+
+        # データセットを準備する
+        if args.dataset_class is None:
+            blueprint_generator = BlueprintGenerator(ConfigSanitizer(True, True, False, False))
+            if args.dataset_config is not None:
+                accelerator.print(f"Load dataset config from {args.dataset_config}")
+                user_config = config_util.load_user_config(args.dataset_config)
+                ignored = ["train_data_dir", "reg_data_dir", "in_json"]
+                if any(getattr(args, attr) is not None for attr in ignored):
+                    accelerator.print(
+                        "ignore following options because config file is found: {0} / 設定ファイルが利用されるため以下のオプションは無視されます: {0}".format(
+                            ", ".join(ignored)
+                        )
+                    )
+            else:
+                use_dreambooth_method = args.in_json is None
+                if use_dreambooth_method:
+                    accelerator.print("Use DreamBooth method.")
+                    user_config = {
+                        "datasets": [
+                            {
+                                "subsets": config_util.generate_dreambooth_subsets_config_by_subdirs(
+                                    args.train_data_dir, args.reg_data_dir
+                                )
+                            }
+                        ]
+                    }
+                else:
+                    print("Train with captions.")
+                    user_config = {
+                        "datasets": [
+                            {
+                                "subsets": [
+                                    {
+                                        "image_dir": args.train_data_dir,
+                                        "metadata_file": args.in_json,
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+
+            blueprint = blueprint_generator.generate(user_config, args, tokenizer=tokenizer_or_list)
+            train_dataset_group = config_util.generate_dataset_group_by_blueprint(blueprint.dataset_group)
+        else:
+            train_dataset_group = train_util.load_arbitrary_dataset(args, tokenizer_or_list)
+
+        self.assert_extra_args(args, train_dataset_group)
+
+        current_epoch = Value("i", 0)
+        current_step = Value("i", 0)
+        ds_for_collater = train_dataset_group if args.max_data_loader_n_workers == 0 else None
+        collater = train_util.collater_class(current_epoch, current_step, ds_for_collater)
+
+        # make captions: tokenstring tokenstring1 tokenstring2 ...tokenstringn という文字列に書き換える超乱暴な実装
+        if use_template:
+            accelerator.print(f"use template for training captions. is object: {args.use_object_template}")
+            templates = imagenet_templates_small if args.use_object_template else imagenet_style_templates_small
+            replace_to = " ".join(token_strings)
+            captions = []
+            for tmpl in templates:
+                captions.append(tmpl.format(replace_to))
+            train_dataset_group.add_replacement("", captions)
+
+            # サンプル生成用
+            if args.num_vectors_per_token > 1:
+                prompt_replacement = (args.token_string, replace_to)
+            else:
+                prompt_replacement = None
+        else:
+            # サンプル生成用
+            if args.num_vectors_per_token > 1:
+                replace_to = " ".join(token_strings)
+                train_dataset_group.add_replacement(args.token_string, replace_to)
+                prompt_replacement = (args.token_string, replace_to)
+            else:
+                prompt_replacement = None
+
+        if args.debug_dataset:
+            train_util.debug_dataset(train_dataset_group, show_input_ids=True)
+            return
+        if len(train_dataset_group) == 0:
+            accelerator.print("No data found. Please verify arguments / 画像がありません。引数指定を確認してください")
+            return
+
+        if cache_latents:
+            assert (
+                train_dataset_group.is_latent_cacheable()
+            ), "when caching latents, either color_aug or random_crop cannot be used / latentをキャッシュするときはcolor_augとrandom_cropは使えません"
+
+        # モデルに xformers とか memory efficient attention を組み込む
+        train_util.replace_unet_modules(unet, args.mem_eff_attn, args.xformers, args.sdpa)
+        vae.set_use_memory_efficient_attention_xformers(args.xformers)
+
+        # 学習を準備する
+        if cache_latents:
+            vae.to(accelerator.device, dtype=vae_dtype)
+            vae.requires_grad_(False)
+            vae.eval()
+            with torch.no_grad():
+                train_dataset_group.cache_latents(vae, args.vae_batch_size, args.cache_latents_to_disk, accelerator.is_main_process)
+            vae.to("cpu")
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+            gc.collect()
+
+            accelerator.wait_for_everyone()
+
+        if args.gradient_checkpointing:
+            unet.enable_gradient_checkpointing()
+            for text_encoder in text_encoders:
+                text_encoder.gradient_checkpointing_enable()
+
+        # 学習に必要なクラスを準備する
+        accelerator.print("prepare optimizer, data loader etc.")
+        trainable_params = []
+        for text_encoder in text_encoders:
+            trainable_params += text_encoder.get_input_embeddings().parameters()
+        _, _, optimizer = train_util.get_optimizer(args, trainable_params)
+
+        # dataloaderを準備する
+        # DataLoaderのプロセス数：0はメインプロセスになる
+        n_workers = min(args.max_data_loader_n_workers, os.cpu_count() - 1)  # cpu_count-1 ただし最大で指定された数まで
+        train_dataloader = torch.utils.data.DataLoader(
+            train_dataset_group,
+            batch_size=1,
+            shuffle=True,
+            collate_fn=collater,
+            num_workers=n_workers,
+            persistent_workers=args.persistent_data_loader_workers,
+        )
+
+        # 学習ステップ数を計算する
+        if args.max_train_epochs is not None:
+            args.max_train_steps = args.max_train_epochs * math.ceil(
+                len(train_dataloader) / accelerator.num_processes / args.gradient_accumulation_steps
+            )
+            accelerator.print(
+                f"override steps. steps for {args.max_train_epochs} epochs is / 指定エポックまでのステップ数: {args.max_train_steps}"
+            )
+
+        # データセット側にも学習ステップを送信
+        train_dataset_group.set_max_train_steps(args.max_train_steps)
+
+        # lr schedulerを用意する
+        lr_scheduler = train_util.get_scheduler_fix(args, optimizer, accelerator.num_processes)
+
+        # acceleratorがなんかよろしくやってくれるらしい
+        if len(text_encoders) == 1:
+            text_encoder_or_list, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+                text_encoder_or_list, optimizer, train_dataloader, lr_scheduler
+            )
+            # transform DDP after prepare
+            text_encoder_or_list, unet = train_util.transform_if_model_is_DDP(text_encoder_or_list, unet)
+
+        elif len(text_encoders) == 2:
+            text_encoder1, text_encoder2, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+                text_encoders[0], text_encoders[1], optimizer, train_dataloader, lr_scheduler
+            )
+            # transform DDP after prepare
+            text_encoder1, text_encoder2, unet = train_util.transform_if_model_is_DDP(text_encoder1, text_encoder2, unet)
+
+            text_encoder_or_list = text_encoders = [text_encoder1, text_encoder2]
+
+        else:
+            raise NotImplementedError()
+
+        index_no_updates_list = []
+        orig_embeds_params_list = []
+        for tokenizer, token_ids, text_encoder in zip(tokenizers, token_ids_list, text_encoders):
+            index_no_updates = torch.arange(len(tokenizer)) < token_ids[0]
+            index_no_updates_list.append(index_no_updates)
+
+            # accelerator.print(len(index_no_updates), torch.sum(index_no_updates))
+            orig_embeds_params = accelerator.unwrap_model(text_encoder).get_input_embeddings().weight.data.detach().clone()
+            orig_embeds_params_list.append(orig_embeds_params)
+
+            # Freeze all parameters except for the token embeddings in text encoder
+            text_encoder.requires_grad_(True)
+            text_encoder.text_model.encoder.requires_grad_(False)
+            text_encoder.text_model.final_layer_norm.requires_grad_(False)
+            text_encoder.text_model.embeddings.position_embedding.requires_grad_(False)
+            # text_encoder.text_model.embeddings.token_embedding.requires_grad_(True)
+
+        unet.requires_grad_(False)
+        unet.to(accelerator.device, dtype=weight_dtype)
+        if args.gradient_checkpointing:  # according to TI example in Diffusers, train is required
+            # TODO U-Netをオリジナルに置き換えたのでいらないはずなので、後で確認して消す
+            unet.train()
+        else:
+            unet.eval()
+
+        if not cache_latents:  # キャッシュしない場合はVAEを使うのでVAEを準備する
+            vae.requires_grad_(False)
+            vae.eval()
+            vae.to(accelerator.device, dtype=vae_dtype)
+
+        # 実験的機能：勾配も含めたfp16学習を行う　PyTorchにパッチを当ててfp16でのgrad scaleを有効にする
+        if args.full_fp16:
+            train_util.patch_accelerator_for_fp16_training(accelerator)
+            for text_encoder in text_encoders:
+                text_encoder.to(weight_dtype)
+        if args.full_bf16:
+            for text_encoder in text_encoders:
+                text_encoder.to(weight_dtype)
+
+        # resumeする
+        train_util.resume_from_local_or_hf_if_specified(accelerator, args)
+
+        # epoch数を計算する
+        num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+        num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+        if (args.save_n_epoch_ratio is not None) and (args.save_n_epoch_ratio > 0):
+            args.save_every_n_epochs = math.floor(num_train_epochs / args.save_n_epoch_ratio) or 1
+
+        # 学習する
+        total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
+        accelerator.print("running training / 学習開始")
+        accelerator.print(f"  num train images * repeats / 学習画像の数×繰り返し回数: {train_dataset_group.num_train_images}")
+        accelerator.print(f"  num reg images / 正則化画像の数: {train_dataset_group.num_reg_images}")
+        accelerator.print(f"  num batches per epoch / 1epochのバッチ数: {len(train_dataloader)}")
+        accelerator.print(f"  num epochs / epoch数: {num_train_epochs}")
+        accelerator.print(f"  batch size per device / バッチサイズ: {args.train_batch_size}")
+        accelerator.print(
+            f"  total train batch size (with parallel & distributed & accumulation) / 総バッチサイズ（並列学習、勾配合計含む）: {total_batch_size}"
+        )
+        accelerator.print(f"  gradient ccumulation steps / 勾配を合計するステップ数 = {args.gradient_accumulation_steps}")
+        accelerator.print(f"  total optimization steps / 学習ステップ数: {args.max_train_steps}")
+
+        progress_bar = tqdm(range(args.max_train_steps), smoothing=0, disable=not accelerator.is_local_main_process, desc="steps")
+        global_step = 0
+
+        noise_scheduler = DDPMScheduler(
+            beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, clip_sample=False
+        )
+        prepare_scheduler_for_custom_training(noise_scheduler, accelerator.device)
+
+        if accelerator.is_main_process:
+            accelerator.init_trackers("textual_inversion" if args.log_tracker_name is None else args.log_tracker_name)
+
+        # function for saving/removing
+        def save_model(ckpt_name, embs_list, steps, epoch_no, force_sync_upload=False):
+            os.makedirs(args.output_dir, exist_ok=True)
+            ckpt_file = os.path.join(args.output_dir, ckpt_name)
+
+            accelerator.print(f"\nsaving checkpoint: {ckpt_file}")
+            self.save_weights(ckpt_file, embs_list, save_dtype)
+            if args.huggingface_repo_id is not None:
+                huggingface_util.upload(args, ckpt_file, "/" + ckpt_name, force_sync_upload=force_sync_upload)
+
+        def remove_model(old_ckpt_name):
+            old_ckpt_file = os.path.join(args.output_dir, old_ckpt_name)
+            if os.path.exists(old_ckpt_file):
+                accelerator.print(f"removing old checkpoint: {old_ckpt_file}")
+                os.remove(old_ckpt_file)
+
+        # training loop
+        for epoch in range(num_train_epochs):
+            accelerator.print(f"\nepoch {epoch+1}/{num_train_epochs}")
+            current_epoch.value = epoch + 1
+
+            for text_encoder in text_encoders:
+                text_encoder.train()
+
+            loss_total = 0
+
+            for step, batch in enumerate(train_dataloader):
+                current_step.value = global_step
+                with accelerator.accumulate(text_encoders[0]):
+                    with torch.no_grad():
+                        if "latents" in batch and batch["latents"] is not None:
+                            latents = batch["latents"].to(accelerator.device)
+                        else:
+                            # latentに変換
+                            latents = vae.encode(batch["images"].to(dtype=vae_dtype)).latent_dist.sample()
+                        latents = latents * self.vae_scale_factor
+
+                    # Get the text embedding for conditioning
+                    text_encoder_conds = self.get_text_cond(args, accelerator, batch, tokenizers, text_encoders, weight_dtype)
+
+                    # Sample noise, sample a random timestep for each image, and add noise to the latents,
+                    # with noise offset and/or multires noise if specified
+                    noise, noisy_latents, timesteps = train_util.get_noise_noisy_latents_and_timesteps(
+                        args, noise_scheduler, latents
+                    )
+
+                    # Predict the noise residual
+                    with accelerator.autocast():
+                        noise_pred = self.call_unet(
+                            args, accelerator, unet, noisy_latents, timesteps, text_encoder_conds, batch, weight_dtype
+                        )
+
+                    if args.v_parameterization:
+                        # v-parameterization training
+                        target = noise_scheduler.get_velocity(latents, noise, timesteps)
+                    else:
+                        target = noise
+
+                    loss = torch.nn.functional.mse_loss(noise_pred.float(), target.float(), reduction="none")
+                    loss = loss.mean([1, 2, 3])
+
+                    loss_weights = batch["loss_weights"]  # 各sampleごとのweight
+                    loss = loss * loss_weights
+
+                    if args.min_snr_gamma:
+                        loss = apply_snr_weight(loss, timesteps, noise_scheduler, args.min_snr_gamma)
+                    if args.scale_v_pred_loss_like_noise_pred:
+                        loss = scale_v_prediction_loss_like_noise_prediction(loss, timesteps, noise_scheduler)
+
+                    loss = loss.mean()  # 平均なのでbatch_sizeで割る必要なし
+
+                    accelerator.backward(loss)
+                    if accelerator.sync_gradients and args.max_grad_norm != 0.0:
+                        params_to_clip = text_encoder.get_input_embeddings().parameters()
+                        accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
+
+                    optimizer.step()
+                    lr_scheduler.step()
+                    optimizer.zero_grad(set_to_none=True)
+
+                    # Let's make sure we don't update any embedding weights besides the newly added token
+                    with torch.no_grad():
+                        for text_encoder, orig_embeds_params, index_no_updates in zip(
+                            text_encoders, orig_embeds_params_list, index_no_updates_list
+                        ):
+                            accelerator.unwrap_model(text_encoder).get_input_embeddings().weight[
+                                index_no_updates
+                            ] = orig_embeds_params[index_no_updates]
+
+                # Checks if the accelerator has performed an optimization step behind the scenes
+                if accelerator.sync_gradients:
+                    progress_bar.update(1)
+                    global_step += 1
+
+                    self.sample_images(
+                        accelerator,
+                        args,
+                        None,
+                        global_step,
+                        accelerator.device,
+                        vae,
+                        tokenizer_or_list,
+                        text_encoder_or_list,
+                        unet,
+                        prompt_replacement,
+                    )
+
+                    # 指定ステップごとにモデルを保存
+                    if args.save_every_n_steps is not None and global_step % args.save_every_n_steps == 0:
+                        accelerator.wait_for_everyone()
+                        if accelerator.is_main_process:
+                            updated_embs_list = []
+                            for text_encoder, token_ids in zip(text_encoders, token_ids_list):
+                                updated_embs = (
+                                    accelerator.unwrap_model(text_encoder)
+                                    .get_input_embeddings()
+                                    .weight[token_ids]
+                                    .data.detach()
+                                    .clone()
+                                )
+                                updated_embs_list.append(updated_embs)
+
+                            ckpt_name = train_util.get_step_ckpt_name(args, "." + args.save_model_as, global_step)
+                            save_model(ckpt_name, updated_embs_list, global_step, epoch)
+
+                            if args.save_state:
+                                train_util.save_and_remove_state_stepwise(args, accelerator, global_step)
+
+                            remove_step_no = train_util.get_remove_step_no(args, global_step)
+                            if remove_step_no is not None:
+                                remove_ckpt_name = train_util.get_step_ckpt_name(args, "." + args.save_model_as, remove_step_no)
+                                remove_model(remove_ckpt_name)
+
+                current_loss = loss.detach().item()
+                if args.logging_dir is not None:
+                    logs = {"loss": current_loss, "lr": float(lr_scheduler.get_last_lr()[0])}
+                    if (
+                        args.optimizer_type.lower().startswith("DAdapt".lower()) or args.optimizer_type.lower() == "Prodigy".lower()
+                    ):  # tracking d*lr value
+                        logs["lr/d*lr"] = (
+                            lr_scheduler.optimizers[0].param_groups[0]["d"] * lr_scheduler.optimizers[0].param_groups[0]["lr"]
+                        )
+                    accelerator.log(logs, step=global_step)
+
+                loss_total += current_loss
+                avr_loss = loss_total / (step + 1)
+                logs = {"loss": avr_loss}  # , "lr": lr_scheduler.get_last_lr()[0]}
+                progress_bar.set_postfix(**logs)
+
+                if global_step >= args.max_train_steps:
+                    break
+
+            if args.logging_dir is not None:
+                logs = {"loss/epoch": loss_total / len(train_dataloader)}
+                accelerator.log(logs, step=epoch + 1)
+
+            accelerator.wait_for_everyone()
+
+            updated_embs_list = []
+            for text_encoder, token_ids in zip(text_encoders, token_ids_list):
+                updated_embs = accelerator.unwrap_model(text_encoder).get_input_embeddings().weight[token_ids].data.detach().clone()
+                updated_embs_list.append(updated_embs)
+
+            if args.save_every_n_epochs is not None:
+                saving = (epoch + 1) % args.save_every_n_epochs == 0 and (epoch + 1) < num_train_epochs
+                if accelerator.is_main_process and saving:
+                    ckpt_name = train_util.get_epoch_ckpt_name(args, "." + args.save_model_as, epoch + 1)
+                    save_model(ckpt_name, updated_embs_list, epoch + 1, global_step)
+
+                    remove_epoch_no = train_util.get_remove_epoch_no(args, epoch + 1)
+                    if remove_epoch_no is not None:
+                        remove_ckpt_name = train_util.get_epoch_ckpt_name(args, "." + args.save_model_as, remove_epoch_no)
+                        remove_model(remove_ckpt_name)
+
+                    if args.save_state:
+                        train_util.save_and_remove_state_on_epoch_end(args, accelerator, epoch + 1)
+
+            self.sample_images(
+                accelerator,
+                args,
+                epoch + 1,
+                global_step,
+                accelerator.device,
+                vae,
+                tokenizer_or_list,
+                text_encoder_or_list,
+                unet,
+                prompt_replacement,
+            )
+
+            # end of epoch
+
+        is_main_process = accelerator.is_main_process
+        if is_main_process:
+            text_encoder = accelerator.unwrap_model(text_encoder)
+
+        accelerator.end_training()
+
+        if args.save_state and is_main_process:
+            train_util.save_state_on_train_end(args, accelerator)
+
+        updated_embs = text_encoder.get_input_embeddings().weight[token_ids].data.detach().clone()
+
+        if is_main_process:
+            ckpt_name = train_util.get_last_ckpt_name(args, "." + args.save_model_as)
+            save_model(ckpt_name, updated_embs_list, global_step, num_train_epochs, force_sync_upload=True)
+
+            print("model saved.")
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    train_util.add_sd_models_arguments(parser)
+    train_util.add_dataset_arguments(parser, True, True, False)
+    train_util.add_training_arguments(parser, True)
+    train_util.add_optimizer_arguments(parser)
+    config_util.add_config_arguments(parser)
+    custom_train_functions.add_custom_train_arguments(parser, False)
+
+    parser.add_argument(
+        "--save_model_as",
+        type=str,
+        default="pt",
+        choices=[None, "ckpt", "pt", "safetensors"],
+        help="format to save the model (default is .pt) / モデル保存時の形式（デフォルトはpt）",
+    )
+
+    parser.add_argument("--weights", type=str, default=None, help="embedding weights to initialize / 学習するネットワークの初期重み")
+    parser.add_argument(
+        "--num_vectors_per_token", type=int, default=1, help="number of vectors per token / トークンに割り当てるembeddingsの要素数"
+    )
+    parser.add_argument(
+        "--token_string",
+        type=str,
+        default=None,
+        help="token string used in training, must not exist in tokenizer / 学習時に使用されるトークン文字列、tokenizerに存在しない文字であること",
+    )
+    parser.add_argument("--init_word", type=str, default=None, help="words to initialize vector / ベクトルを初期化に使用する単語、複数可")
+    parser.add_argument(
+        "--use_object_template",
+        action="store_true",
+        help="ignore caption and use default templates for object / キャプションは使わずデフォルトの物体用テンプレートで学習する",
+    )
+    parser.add_argument(
+        "--use_style_template",
+        action="store_true",
+        help="ignore caption and use default templates for stype / キャプションは使わずデフォルトのスタイル用テンプレートで学習する",
+    )
+    parser.add_argument(
+        "--no_half_vae",
+        action="store_true",
+        help="do not use fp16/bf16 VAE in mixed precision (use float VAE) / mixed precisionでも fp16/bf16 VAEを使わずfloat VAEを使う",
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    args = train_util.read_config_from_file(args, parser)
+
+    trainer = TextualInversionTrainer()
+    trainer.train(args)
diff --git a/train_textual_inversion_XTI.py b/train_textual_inversion_XTI.py
new file mode 100644
index 0000000000000000000000000000000000000000..0e91c71c35876df372403fbc53a5fb7eaf714974
--- /dev/null
+++ b/train_textual_inversion_XTI.py
@@ -0,0 +1,684 @@
+import importlib
+import argparse
+import gc
+import math
+import os
+import toml
+from multiprocessing import Value
+
+from tqdm import tqdm
+import torch
+from accelerate.utils import set_seed
+import diffusers
+from diffusers import DDPMScheduler
+import library
+
+import library.train_util as train_util
+import library.huggingface_util as huggingface_util
+import library.config_util as config_util
+from library.config_util import (
+    ConfigSanitizer,
+    BlueprintGenerator,
+)
+import library.custom_train_functions as custom_train_functions
+from library.custom_train_functions import (
+    apply_snr_weight,
+    prepare_scheduler_for_custom_training,
+    pyramid_noise_like,
+    apply_noise_offset,
+    scale_v_prediction_loss_like_noise_prediction,
+)
+import library.original_unet as original_unet
+from XTI_hijack import unet_forward_XTI, downblock_forward_XTI, upblock_forward_XTI
+
+imagenet_templates_small = [
+    "a photo of a {}",
+    "a rendering of a {}",
+    "a cropped photo of the {}",
+    "the photo of a {}",
+    "a photo of a clean {}",
+    "a photo of a dirty {}",
+    "a dark photo of the {}",
+    "a photo of my {}",
+    "a photo of the cool {}",
+    "a close-up photo of a {}",
+    "a bright photo of the {}",
+    "a cropped photo of a {}",
+    "a photo of the {}",
+    "a good photo of the {}",
+    "a photo of one {}",
+    "a close-up photo of the {}",
+    "a rendition of the {}",
+    "a photo of the clean {}",
+    "a rendition of a {}",
+    "a photo of a nice {}",
+    "a good photo of a {}",
+    "a photo of the nice {}",
+    "a photo of the small {}",
+    "a photo of the weird {}",
+    "a photo of the large {}",
+    "a photo of a cool {}",
+    "a photo of a small {}",
+]
+
+imagenet_style_templates_small = [
+    "a painting in the style of {}",
+    "a rendering in the style of {}",
+    "a cropped painting in the style of {}",
+    "the painting in the style of {}",
+    "a clean painting in the style of {}",
+    "a dirty painting in the style of {}",
+    "a dark painting in the style of {}",
+    "a picture in the style of {}",
+    "a cool painting in the style of {}",
+    "a close-up painting in the style of {}",
+    "a bright painting in the style of {}",
+    "a cropped painting in the style of {}",
+    "a good painting in the style of {}",
+    "a close-up painting in the style of {}",
+    "a rendition in the style of {}",
+    "a nice painting in the style of {}",
+    "a small painting in the style of {}",
+    "a weird painting in the style of {}",
+    "a large painting in the style of {}",
+]
+
+
+def train(args):
+    if args.output_name is None:
+        args.output_name = args.token_string
+    use_template = args.use_object_template or args.use_style_template
+
+    train_util.verify_training_args(args)
+    train_util.prepare_dataset_args(args, True)
+
+    if args.sample_every_n_steps is not None or args.sample_every_n_epochs is not None:
+        print(
+            "sample_every_n_steps and sample_every_n_epochs are not supported in this script currently / sample_every_n_stepsとsample_every_n_epochsは現在このスクリプトではサポートされていません"
+        )
+    assert (
+        args.dataset_class is None
+    ), "dataset_class is not supported in this script currently / dataset_classは現在このスクリプトではサポートされていません"
+
+    cache_latents = args.cache_latents
+
+    if args.seed is not None:
+        set_seed(args.seed)
+
+    tokenizer = train_util.load_tokenizer(args)
+
+    # acceleratorを準備する
+    print("prepare accelerator")
+    accelerator = train_util.prepare_accelerator(args)
+
+    # mixed precisionに対応した型を用意しておき適宜castする
+    weight_dtype, save_dtype = train_util.prepare_dtype(args)
+
+    # モデルを読み込む
+    text_encoder, vae, unet, _ = train_util.load_target_model(args, weight_dtype, accelerator)
+
+    # Convert the init_word to token_id
+    if args.init_word is not None:
+        init_token_ids = tokenizer.encode(args.init_word, add_special_tokens=False)
+        if len(init_token_ids) > 1 and len(init_token_ids) != args.num_vectors_per_token:
+            print(
+                f"token length for init words is not same to num_vectors_per_token, init words is repeated or truncated / 初期化単語のトークン長がnum_vectors_per_tokenと合わないため、繰り返しまたは切り捨てが発生します: length {len(init_token_ids)}"
+            )
+    else:
+        init_token_ids = None
+
+    # add new word to tokenizer, count is num_vectors_per_token
+    token_strings = [args.token_string] + [f"{args.token_string}{i+1}" for i in range(args.num_vectors_per_token - 1)]
+    num_added_tokens = tokenizer.add_tokens(token_strings)
+    assert (
+        num_added_tokens == args.num_vectors_per_token
+    ), f"tokenizer has same word to token string. please use another one / 指定したargs.token_stringは既に存在します。別の単語を使ってください: {args.token_string}"
+
+    token_ids = tokenizer.convert_tokens_to_ids(token_strings)
+    print(f"tokens are added: {token_ids}")
+    assert min(token_ids) == token_ids[0] and token_ids[-1] == token_ids[0] + len(token_ids) - 1, f"token ids is not ordered"
+    assert len(tokenizer) - 1 == token_ids[-1], f"token ids is not end of tokenize: {len(tokenizer)}"
+
+    token_strings_XTI = []
+    XTI_layers = [
+        "IN01",
+        "IN02",
+        "IN04",
+        "IN05",
+        "IN07",
+        "IN08",
+        "MID",
+        "OUT03",
+        "OUT04",
+        "OUT05",
+        "OUT06",
+        "OUT07",
+        "OUT08",
+        "OUT09",
+        "OUT10",
+        "OUT11",
+    ]
+    for layer_name in XTI_layers:
+        token_strings_XTI += [f"{t}_{layer_name}" for t in token_strings]
+
+    tokenizer.add_tokens(token_strings_XTI)
+    token_ids_XTI = tokenizer.convert_tokens_to_ids(token_strings_XTI)
+    print(f"tokens are added (XTI): {token_ids_XTI}")
+    # Resize the token embeddings as we are adding new special tokens to the tokenizer
+    text_encoder.resize_token_embeddings(len(tokenizer))
+
+    # Initialise the newly added placeholder token with the embeddings of the initializer token
+    token_embeds = text_encoder.get_input_embeddings().weight.data
+    if init_token_ids is not None:
+        for i, token_id in enumerate(token_ids_XTI):
+            token_embeds[token_id] = token_embeds[init_token_ids[(i // 16) % len(init_token_ids)]]
+            # print(token_id, token_embeds[token_id].mean(), token_embeds[token_id].min())
+
+    # load weights
+    if args.weights is not None:
+        embeddings = load_weights(args.weights)
+        assert len(token_ids) == len(
+            embeddings
+        ), f"num_vectors_per_token is mismatch for weights / 指定した重みとnum_vectors_per_tokenの値が異なります: {len(embeddings)}"
+        # print(token_ids, embeddings.size())
+        for token_id, embedding in zip(token_ids_XTI, embeddings):
+            token_embeds[token_id] = embedding
+            # print(token_id, token_embeds[token_id].mean(), token_embeds[token_id].min())
+        print(f"weighs loaded")
+
+    print(f"create embeddings for {args.num_vectors_per_token} tokens, for {args.token_string}")
+
+    # データセットを準備する
+    blueprint_generator = BlueprintGenerator(ConfigSanitizer(True, True, False, False))
+    if args.dataset_config is not None:
+        print(f"Load dataset config from {args.dataset_config}")
+        user_config = config_util.load_user_config(args.dataset_config)
+        ignored = ["train_data_dir", "reg_data_dir", "in_json"]
+        if any(getattr(args, attr) is not None for attr in ignored):
+            print(
+                "ignore following options because config file is found: {0} / 設定ファイルが利用されるため以下のオプションは無視されます: {0}".format(
+                    ", ".join(ignored)
+                )
+            )
+    else:
+        use_dreambooth_method = args.in_json is None
+        if use_dreambooth_method:
+            print("Use DreamBooth method.")
+            user_config = {
+                "datasets": [
+                    {"subsets": config_util.generate_dreambooth_subsets_config_by_subdirs(args.train_data_dir, args.reg_data_dir)}
+                ]
+            }
+        else:
+            print("Train with captions.")
+            user_config = {
+                "datasets": [
+                    {
+                        "subsets": [
+                            {
+                                "image_dir": args.train_data_dir,
+                                "metadata_file": args.in_json,
+                            }
+                        ]
+                    }
+                ]
+            }
+
+    blueprint = blueprint_generator.generate(user_config, args, tokenizer=tokenizer)
+    train_dataset_group = config_util.generate_dataset_group_by_blueprint(blueprint.dataset_group)
+    train_dataset_group.enable_XTI(XTI_layers, token_strings=token_strings)
+    current_epoch = Value("i", 0)
+    current_step = Value("i", 0)
+    ds_for_collater = train_dataset_group if args.max_data_loader_n_workers == 0 else None
+    collater = train_util.collater_class(current_epoch, current_step, ds_for_collater)
+
+    # make captions: tokenstring tokenstring1 tokenstring2 ...tokenstringn という文字列に書き換える超乱暴な実装
+    if use_template:
+        print(f"use template for training captions. is object: {args.use_object_template}")
+        templates = imagenet_templates_small if args.use_object_template else imagenet_style_templates_small
+        replace_to = " ".join(token_strings)
+        captions = []
+        for tmpl in templates:
+            captions.append(tmpl.format(replace_to))
+        train_dataset_group.add_replacement("", captions)
+
+        if args.num_vectors_per_token > 1:
+            prompt_replacement = (args.token_string, replace_to)
+        else:
+            prompt_replacement = None
+    else:
+        if args.num_vectors_per_token > 1:
+            replace_to = " ".join(token_strings)
+            train_dataset_group.add_replacement(args.token_string, replace_to)
+            prompt_replacement = (args.token_string, replace_to)
+        else:
+            prompt_replacement = None
+
+    if args.debug_dataset:
+        train_util.debug_dataset(train_dataset_group, show_input_ids=True)
+        return
+    if len(train_dataset_group) == 0:
+        print("No data found. Please verify arguments / 画像がありません。引数指定を確認してください")
+        return
+
+    if cache_latents:
+        assert (
+            train_dataset_group.is_latent_cacheable()
+        ), "when caching latents, either color_aug or random_crop cannot be used / latentをキャッシュするときはcolor_augとrandom_cropは使えません"
+
+    # モデルに xformers とか memory efficient attention を組み込む
+    train_util.replace_unet_modules(unet, args.mem_eff_attn, args.xformers, args.sdpa)
+    original_unet.UNet2DConditionModel.forward = unet_forward_XTI
+    original_unet.CrossAttnDownBlock2D.forward = downblock_forward_XTI
+    original_unet.CrossAttnUpBlock2D.forward = upblock_forward_XTI
+
+    # 学習を準備する
+    if cache_latents:
+        vae.to(accelerator.device, dtype=weight_dtype)
+        vae.requires_grad_(False)
+        vae.eval()
+        with torch.no_grad():
+            train_dataset_group.cache_latents(vae, args.vae_batch_size, args.cache_latents_to_disk, accelerator.is_main_process)
+        vae.to("cpu")
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        gc.collect()
+
+        accelerator.wait_for_everyone()
+
+    if args.gradient_checkpointing:
+        unet.enable_gradient_checkpointing()
+        text_encoder.gradient_checkpointing_enable()
+
+    # 学習に必要なクラスを準備する
+    print("prepare optimizer, data loader etc.")
+    trainable_params = text_encoder.get_input_embeddings().parameters()
+    _, _, optimizer = train_util.get_optimizer(args, trainable_params)
+
+    # dataloaderを準備する
+    # DataLoaderのプロセス数：0はメインプロセスになる
+    n_workers = min(args.max_data_loader_n_workers, os.cpu_count() - 1)  # cpu_count-1 ただし最大で指定された数まで
+    train_dataloader = torch.utils.data.DataLoader(
+        train_dataset_group,
+        batch_size=1,
+        shuffle=True,
+        collate_fn=collater,
+        num_workers=n_workers,
+        persistent_workers=args.persistent_data_loader_workers,
+    )
+
+    # 学習ステップ数を計算する
+    if args.max_train_epochs is not None:
+        args.max_train_steps = args.max_train_epochs * math.ceil(
+            len(train_dataloader) / accelerator.num_processes / args.gradient_accumulation_steps
+        )
+        print(f"override steps. steps for {args.max_train_epochs} epochs is / 指定エポックまでのステップ数: {args.max_train_steps}")
+
+    # データセット側にも学習ステップを送信
+    train_dataset_group.set_max_train_steps(args.max_train_steps)
+
+    # lr schedulerを用意する
+    lr_scheduler = train_util.get_scheduler_fix(args, optimizer, accelerator.num_processes)
+
+    # acceleratorがなんかよろしくやってくれるらしい
+    text_encoder, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+        text_encoder, optimizer, train_dataloader, lr_scheduler
+    )
+
+    # transform DDP after prepare
+    text_encoder, unet = train_util.transform_if_model_is_DDP(text_encoder, unet)
+
+    index_no_updates = torch.arange(len(tokenizer)) < token_ids_XTI[0]
+    # print(len(index_no_updates), torch.sum(index_no_updates))
+    orig_embeds_params = accelerator.unwrap_model(text_encoder).get_input_embeddings().weight.data.detach().clone()
+
+    # Freeze all parameters except for the token embeddings in text encoder
+    text_encoder.requires_grad_(True)
+    text_encoder.text_model.encoder.requires_grad_(False)
+    text_encoder.text_model.final_layer_norm.requires_grad_(False)
+    text_encoder.text_model.embeddings.position_embedding.requires_grad_(False)
+    # text_encoder.text_model.embeddings.token_embedding.requires_grad_(True)
+
+    unet.requires_grad_(False)
+    unet.to(accelerator.device, dtype=weight_dtype)
+    if args.gradient_checkpointing:  # according to TI example in Diffusers, train is required
+        unet.train()
+    else:
+        unet.eval()
+
+    if not cache_latents:
+        vae.requires_grad_(False)
+        vae.eval()
+        vae.to(accelerator.device, dtype=weight_dtype)
+
+    # 実験的機能：勾配も含めたfp16学習を行う　PyTorchにパッチを当ててfp16でのgrad scaleを有効にする
+    if args.full_fp16:
+        train_util.patch_accelerator_for_fp16_training(accelerator)
+        text_encoder.to(weight_dtype)
+
+    # resumeする
+    train_util.resume_from_local_or_hf_if_specified(accelerator, args)
+
+    # epoch数を計算する
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+    if (args.save_n_epoch_ratio is not None) and (args.save_n_epoch_ratio > 0):
+        args.save_every_n_epochs = math.floor(num_train_epochs / args.save_n_epoch_ratio) or 1
+
+    # 学習する
+    total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
+    print("running training / 学習開始")
+    print(f"  num train images * repeats / 学習画像の数×繰り返し回数: {train_dataset_group.num_train_images}")
+    print(f"  num reg images / 正則化画像の数: {train_dataset_group.num_reg_images}")
+    print(f"  num batches per epoch / 1epochのバッチ数: {len(train_dataloader)}")
+    print(f"  num epochs / epoch数: {num_train_epochs}")
+    print(f"  batch size per device / バッチサイズ: {args.train_batch_size}")
+    print(f"  total train batch size (with parallel & distributed & accumulation) / 総バッチサイズ（並列学習、勾配合計含む）: {total_batch_size}")
+    print(f"  gradient ccumulation steps / 勾配を合計するステップ数 = {args.gradient_accumulation_steps}")
+    print(f"  total optimization steps / 学習ステップ数: {args.max_train_steps}")
+
+    progress_bar = tqdm(range(args.max_train_steps), smoothing=0, disable=not accelerator.is_local_main_process, desc="steps")
+    global_step = 0
+
+    noise_scheduler = DDPMScheduler(
+        beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, clip_sample=False
+    )
+    prepare_scheduler_for_custom_training(noise_scheduler, accelerator.device)
+
+    if accelerator.is_main_process:
+        accelerator.init_trackers("textual_inversion" if args.log_tracker_name is None else args.log_tracker_name)
+
+    # function for saving/removing
+    def save_model(ckpt_name, embs, steps, epoch_no, force_sync_upload=False):
+        os.makedirs(args.output_dir, exist_ok=True)
+        ckpt_file = os.path.join(args.output_dir, ckpt_name)
+
+        print(f"\nsaving checkpoint: {ckpt_file}")
+        save_weights(ckpt_file, embs, save_dtype)
+        if args.huggingface_repo_id is not None:
+            huggingface_util.upload(args, ckpt_file, "/" + ckpt_name, force_sync_upload=force_sync_upload)
+
+    def remove_model(old_ckpt_name):
+        old_ckpt_file = os.path.join(args.output_dir, old_ckpt_name)
+        if os.path.exists(old_ckpt_file):
+            print(f"removing old checkpoint: {old_ckpt_file}")
+            os.remove(old_ckpt_file)
+
+    # training loop
+    for epoch in range(num_train_epochs):
+        print(f"\nepoch {epoch+1}/{num_train_epochs}")
+        current_epoch.value = epoch + 1
+
+        text_encoder.train()
+
+        loss_total = 0
+
+        for step, batch in enumerate(train_dataloader):
+            current_step.value = global_step
+            with accelerator.accumulate(text_encoder):
+                with torch.no_grad():
+                    if "latents" in batch and batch["latents"] is not None:
+                        latents = batch["latents"].to(accelerator.device)
+                    else:
+                        # latentに変換
+                        latents = vae.encode(batch["images"].to(dtype=weight_dtype)).latent_dist.sample()
+                    latents = latents * 0.18215
+                b_size = latents.shape[0]
+
+                # Get the text embedding for conditioning
+                input_ids = batch["input_ids"].to(accelerator.device)
+                # weight_dtype) use float instead of fp16/bf16 because text encoder is float
+                encoder_hidden_states = torch.stack(
+                    [
+                        train_util.get_hidden_states(args, s, tokenizer, text_encoder, weight_dtype)
+                        for s in torch.split(input_ids, 1, dim=1)
+                    ]
+                )
+
+                # Sample noise, sample a random timestep for each image, and add noise to the latents,
+                # with noise offset and/or multires noise if specified
+                noise, noisy_latents, timesteps = train_util.get_noise_noisy_latents_and_timesteps(args, noise_scheduler, latents)
+
+                # Predict the noise residual
+                with accelerator.autocast():
+                    noise_pred = unet(noisy_latents, timesteps, encoder_hidden_states=encoder_hidden_states).sample
+
+                if args.v_parameterization:
+                    # v-parameterization training
+                    target = noise_scheduler.get_velocity(latents, noise, timesteps)
+                else:
+                    target = noise
+
+                loss = torch.nn.functional.mse_loss(noise_pred.float(), target.float(), reduction="none")
+                loss = loss.mean([1, 2, 3])
+
+                loss_weights = batch["loss_weights"]  # 各sampleごとのweight
+
+                loss = loss * loss_weights
+                if args.min_snr_gamma:
+                    loss = apply_snr_weight(loss, timesteps, noise_scheduler, args.min_snr_gamma)
+                if args.scale_v_pred_loss_like_noise_pred:
+                    loss = scale_v_prediction_loss_like_noise_prediction(loss, timesteps, noise_scheduler)
+
+                loss = loss.mean()  # 平均なのでbatch_sizeで割る必要なし
+
+                accelerator.backward(loss)
+                if accelerator.sync_gradients and args.max_grad_norm != 0.0:
+                    params_to_clip = text_encoder.get_input_embeddings().parameters()
+                    accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
+
+                optimizer.step()
+                lr_scheduler.step()
+                optimizer.zero_grad(set_to_none=True)
+
+                # Let's make sure we don't update any embedding weights besides the newly added token
+                with torch.no_grad():
+                    accelerator.unwrap_model(text_encoder).get_input_embeddings().weight[index_no_updates] = orig_embeds_params[
+                        index_no_updates
+                    ]
+
+            # Checks if the accelerator has performed an optimization step behind the scenes
+            if accelerator.sync_gradients:
+                progress_bar.update(1)
+                global_step += 1
+                # TODO: fix sample_images
+                # train_util.sample_images(
+                #     accelerator, args, None, global_step, accelerator.device, vae, tokenizer, text_encoder, unet, prompt_replacement
+                # )
+
+                # 指定ステップごとにモデルを保存
+                if args.save_every_n_steps is not None and global_step % args.save_every_n_steps == 0:
+                    accelerator.wait_for_everyone()
+                    if accelerator.is_main_process:
+                        updated_embs = (
+                            accelerator.unwrap_model(text_encoder)
+                            .get_input_embeddings()
+                            .weight[token_ids_XTI]
+                            .data.detach()
+                            .clone()
+                        )
+
+                        ckpt_name = train_util.get_step_ckpt_name(args, "." + args.save_model_as, global_step)
+                        save_model(ckpt_name, updated_embs, global_step, epoch)
+
+                        if args.save_state:
+                            train_util.save_and_remove_state_stepwise(args, accelerator, global_step)
+
+                        remove_step_no = train_util.get_remove_step_no(args, global_step)
+                        if remove_step_no is not None:
+                            remove_ckpt_name = train_util.get_step_ckpt_name(args, "." + args.save_model_as, remove_step_no)
+                            remove_model(remove_ckpt_name)
+
+            current_loss = loss.detach().item()
+            if args.logging_dir is not None:
+                logs = {"loss": current_loss, "lr": float(lr_scheduler.get_last_lr()[0])}
+                if (
+                    args.optimizer_type.lower().startswith("DAdapt".lower()) or args.optimizer_type.lower() == "Prodigy".lower()
+                ):  # tracking d*lr value
+                    logs["lr/d*lr"] = (
+                        lr_scheduler.optimizers[0].param_groups[0]["d"] * lr_scheduler.optimizers[0].param_groups[0]["lr"]
+                    )
+                accelerator.log(logs, step=global_step)
+
+            loss_total += current_loss
+            avr_loss = loss_total / (step + 1)
+            logs = {"loss": avr_loss}  # , "lr": lr_scheduler.get_last_lr()[0]}
+            progress_bar.set_postfix(**logs)
+
+            if global_step >= args.max_train_steps:
+                break
+
+        if args.logging_dir is not None:
+            logs = {"loss/epoch": loss_total / len(train_dataloader)}
+            accelerator.log(logs, step=epoch + 1)
+
+        accelerator.wait_for_everyone()
+
+        updated_embs = accelerator.unwrap_model(text_encoder).get_input_embeddings().weight[token_ids_XTI].data.detach().clone()
+
+        if args.save_every_n_epochs is not None:
+            saving = (epoch + 1) % args.save_every_n_epochs == 0 and (epoch + 1) < num_train_epochs
+            if accelerator.is_main_process and saving:
+                ckpt_name = train_util.get_epoch_ckpt_name(args, "." + args.save_model_as, epoch + 1)
+                save_model(ckpt_name, updated_embs, epoch + 1, global_step)
+
+                remove_epoch_no = train_util.get_remove_epoch_no(args, epoch + 1)
+                if remove_epoch_no is not None:
+                    remove_ckpt_name = train_util.get_epoch_ckpt_name(args, "." + args.save_model_as, remove_epoch_no)
+                    remove_model(remove_ckpt_name)
+
+                if args.save_state:
+                    train_util.save_and_remove_state_on_epoch_end(args, accelerator, epoch + 1)
+
+        # TODO: fix sample_images
+        # train_util.sample_images(
+        #     accelerator, args, epoch + 1, global_step, accelerator.device, vae, tokenizer, text_encoder, unet, prompt_replacement
+        # )
+
+        # end of epoch
+
+    is_main_process = accelerator.is_main_process
+    if is_main_process:
+        text_encoder = accelerator.unwrap_model(text_encoder)
+
+    accelerator.end_training()
+
+    if args.save_state and is_main_process:
+        train_util.save_state_on_train_end(args, accelerator)
+
+    updated_embs = text_encoder.get_input_embeddings().weight[token_ids_XTI].data.detach().clone()
+
+    del accelerator  # この後メモリを使うのでこれは消す
+
+    if is_main_process:
+        ckpt_name = train_util.get_last_ckpt_name(args, "." + args.save_model_as)
+        save_model(ckpt_name, updated_embs, global_step, num_train_epochs, force_sync_upload=True)
+
+        print("model saved.")
+
+
+def save_weights(file, updated_embs, save_dtype):
+    updated_embs = updated_embs.reshape(16, -1, updated_embs.shape[-1])
+    updated_embs = updated_embs.chunk(16)
+    XTI_layers = [
+        "IN01",
+        "IN02",
+        "IN04",
+        "IN05",
+        "IN07",
+        "IN08",
+        "MID",
+        "OUT03",
+        "OUT04",
+        "OUT05",
+        "OUT06",
+        "OUT07",
+        "OUT08",
+        "OUT09",
+        "OUT10",
+        "OUT11",
+    ]
+    state_dict = {}
+    for i, layer_name in enumerate(XTI_layers):
+        state_dict[layer_name] = updated_embs[i].squeeze(0).detach().clone().to("cpu").to(save_dtype)
+
+    # if save_dtype is not None:
+    #     for key in list(state_dict.keys()):
+    #         v = state_dict[key]
+    #         v = v.detach().clone().to("cpu").to(save_dtype)
+    #         state_dict[key] = v
+
+    if os.path.splitext(file)[1] == ".safetensors":
+        from safetensors.torch import save_file
+
+        save_file(state_dict, file)
+    else:
+        torch.save(state_dict, file)  # can be loaded in Web UI
+
+
+def load_weights(file):
+    if os.path.splitext(file)[1] == ".safetensors":
+        from safetensors.torch import load_file
+
+        data = load_file(file)
+    else:
+        raise ValueError(f"NOT XTI: {file}")
+
+    if len(data.values()) != 16:
+        raise ValueError(f"NOT XTI: {file}")
+
+    emb = torch.concat([x for x in data.values()])
+
+    return emb
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    train_util.add_sd_models_arguments(parser)
+    train_util.add_dataset_arguments(parser, True, True, False)
+    train_util.add_training_arguments(parser, True)
+    train_util.add_optimizer_arguments(parser)
+    config_util.add_config_arguments(parser)
+    custom_train_functions.add_custom_train_arguments(parser, False)
+
+    parser.add_argument(
+        "--save_model_as",
+        type=str,
+        default="pt",
+        choices=[None, "ckpt", "pt", "safetensors"],
+        help="format to save the model (default is .pt) / モデル保存時の形式（デフォルトはpt）",
+    )
+
+    parser.add_argument("--weights", type=str, default=None, help="embedding weights to initialize / 学習するネットワークの初期重み")
+    parser.add_argument(
+        "--num_vectors_per_token", type=int, default=1, help="number of vectors per token / トークンに割り当てるembeddingsの要素数"
+    )
+    parser.add_argument(
+        "--token_string",
+        type=str,
+        default=None,
+        help="token string used in training, must not exist in tokenizer / 学習時に使用されるトークン文字列、tokenizerに存在しない文字であること",
+    )
+    parser.add_argument("--init_word", type=str, default=None, help="words to initialize vector / ベクトルを初期化に使用する単語、複数可")
+    parser.add_argument(
+        "--use_object_template",
+        action="store_true",
+        help="ignore caption and use default templates for object / キャプションは使わずデフォルトの物体用テンプレートで学習する",
+    )
+    parser.add_argument(
+        "--use_style_template",
+        action="store_true",
+        help="ignore caption and use default templates for stype / キャプションは使わずデフォルトのスタイル用テンプレートで学習する",
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = setup_parser()
+
+    args = parser.parse_args()
+    args = train_util.read_config_from_file(args, parser)
+
+    train(args)
diff --git a/train_ti_README.md b/train_ti_README.md
new file mode 100644
index 0000000000000000000000000000000000000000..ba03d555870fa1a4093a1d105cf01c3febfec6b9
--- /dev/null
+++ b/train_ti_README.md
@@ -0,0 +1,62 @@
+## About learning Textual Inversion
+
+[Textual Inversion](https://textual-inversion.github.io/). I heavily referenced https://github.com/huggingface/diffusers/tree/main/examples/textual_inversion for the implementation.
+
+The trained model can be used as is on the Web UI.
+
+In addition, it is probably compatible with SD2.x, but it has not been tested at this time.
+
+## Learning method
+
+Use ``train_textual_inversion.py``.
+
+Data preparation is exactly the same as ``train_network.py``, so please refer to [their document](./train_network_README-en.md).
+
+## options
+
+Below is an example command line (DreamBooth technique).
+
+```
+accelerate launch --num_cpu_threads_per_process 1 train_textual_inversion.py
+     --pretrained_model_name_or_path=..\models\model.ckpt
+     --train_data_dir=..\data\db\char1 --output_dir=..\ti_train1
+     --resolution=448,640 --train_batch_size=1 --learning_rate=1e-4
+     --max_train_steps=400 --use_8bit_adam --xformers --mixed_precision=fp16
+     --save_every_n_epochs=1 --save_model_as=safetensors --clip_skip=2 --seed=42 --color_aug
+     --token_string=mychar4 --init_word=cute --num_vectors_per_token=4
+```
+
+``--token_string`` specifies the token string for learning. __The learning prompt should contain this string (eg ``mychar4 1girl`` if token_string is mychar4)__. This string part of the prompt is replaced with a new token for Textual Inversion and learned.
+
+``--debug_dataset`` will display the token id after substitution, so you can check if the token string after ``49408`` exists as shown below. I can confirm.
+
+```
+input ids: tensor([[49406, 49408, 49409, 49410, 49411, 49412, 49413, 49414, 49415, 49407,
+          49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+          49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+          49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+          49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+          49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+          49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407, 49407,
+          49407, 49407, 49407, 49407, 49407, 49407, 49407]])
+```
+
+Words that the tokenizer already has (common words) cannot be used.
+
+In ``--init_word``, specify the string of the copy source token when initializing embeddings. It seems to be a good idea to choose something that has a similar concept to what you want to learn. You cannot specify a character string that becomes two or more tokens.
+
+``--num_vectors_per_token`` specifies how many tokens to use for this training. The higher the number, the more expressive it is, but it consumes more tokens. For example, if num_vectors_per_token=8, then the specified token string will consume 8 tokens (out of the 77 token limit for a typical prompt).
+
+
+In addition, the following options can be specified.
+
+* --weights
+   * Load learned embeddings before learning and learn additionally from there.
+* --use_object_template
+   * Learn with default object template strings (such as ``a photo of a {}``) instead of captions. It will be the same as the official implementation. Captions are ignored.
+* --use_style_template
+   * Learn with default style template strings instead of captions (such as ``a painting in the style of {}``). It will be the same as the official implementation. Captions are ignored.
+
+## Generate with the image generation script in this repository
+
+In gen_img_diffusers.py, specify the learned embeddings file with the ``--textual_inversion_embeddings`` option. Using the filename (without the extension) of the embeddings file at the prompt will apply the embeddings.
\ No newline at end of file
diff --git a/v2_inference/v2-inference-v.yaml b/v2_inference/v2-inference-v.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8ec8dfbfefe94ae8522c93017668fea78d580acf
--- /dev/null
+++ b/v2_inference/v2-inference-v.yaml
@@ -0,0 +1,68 @@
+model:
+  base_learning_rate: 1.0e-4
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    parameterization: "v"
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False # we set this to false because this is an inference only config
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        use_fp16: True
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_head_channels: 64 # need to fix for flash-attn
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          #attn_type: "vanilla-xformers"
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"
diff --git a/v2_inference/v2-inference.yaml b/v2_inference/v2-inference.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..152c4f3c2b36c3b246a9cb10eb8166134b0d2e1c
--- /dev/null
+++ b/v2_inference/v2-inference.yaml
@@ -0,0 +1,67 @@
+model:
+  base_learning_rate: 1.0e-4
+  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False # we set this to false because this is an inference only config
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        use_fp16: True
+        image_size: 32 # unused
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_head_channels: 64 # need to fix for flash-attn
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          #attn_type: "vanilla-xformers"
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"
diff --git a/v2_inference/v2-inpainting-inference.yaml b/v2_inference/v2-inpainting-inference.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..32a9471d71b828c51bcbbabfe34c5f6c8282c803
--- /dev/null
+++ b/v2_inference/v2-inpainting-inference.yaml
@@ -0,0 +1,158 @@
+model:
+  base_learning_rate: 5.0e-05
+  target: ldm.models.diffusion.ddpm.LatentInpaintDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: hybrid
+    scale_factor: 0.18215
+    monitor: val/loss_simple_ema
+    finetune_keys: null
+    use_ema: False
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        image_size: 32 # unused
+        in_channels: 9
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_head_channels: 64 # need to fix for flash-attn
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          #attn_type: "vanilla-xformers"
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+            - 1
+            - 2
+            - 4
+            - 4
+          num_res_blocks: 2
+          attn_resolutions: [ ]
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"
+
+
+data:
+  target: ldm.data.laion.WebDataModuleFromConfig
+  params:
+    tar_base: null  # for concat as in LAION-A
+    p_unsafe_threshold: 0.1
+    filter_word_list: "data/filters.yaml"
+    max_pwatermark: 0.45
+    batch_size: 8
+    num_workers: 6
+    multinode: True
+    min_size: 512
+    train:
+      shards:
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-0/{00000..18699}.tar -"
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-1/{00000..18699}.tar -"
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-2/{00000..18699}.tar -"
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-3/{00000..18699}.tar -"
+        - "pipe:aws s3 cp s3://stability-aws/laion-a-native/part-4/{00000..18699}.tar -"  #{00000-94333}.tar"
+      shuffle: 10000
+      image_key: jpg
+      image_transforms:
+      - target: torchvision.transforms.Resize
+        params:
+          size: 512
+          interpolation: 3
+      - target: torchvision.transforms.RandomCrop
+        params:
+          size: 512
+      postprocess:
+        target: ldm.data.laion.AddMask
+        params:
+          mode: "512train-large"
+          p_drop: 0.25
+    # NOTE use enough shards to avoid empty validation loops in workers
+    validation:
+      shards:
+        - "pipe:aws s3 cp s3://deep-floyd-s3/datasets/laion_cleaned-part5/{93001..94333}.tar - "
+      shuffle: 0
+      image_key: jpg
+      image_transforms:
+      - target: torchvision.transforms.Resize
+        params:
+          size: 512
+          interpolation: 3
+      - target: torchvision.transforms.CenterCrop
+        params:
+          size: 512
+      postprocess:
+        target: ldm.data.laion.AddMask
+        params:
+          mode: "512train-large"
+          p_drop: 0.25
+
+lightning:
+  find_unused_parameters: True
+  modelcheckpoint:
+    params:
+      every_n_train_steps: 5000
+
+  callbacks:
+    metrics_over_trainsteps_checkpoint:
+      params:
+        every_n_train_steps: 10000
+
+    image_logger:
+      target: main.ImageLogger
+      params:
+        enable_autocast: False
+        disabled: False
+        batch_frequency: 1000
+        max_images: 4
+        increase_log_steps: False
+        log_first_step: False
+        log_images_kwargs:
+          use_ema_scope: False
+          inpaint: False
+          plot_progressive_rows: False
+          plot_diffusion_rows: False
+          N: 4
+          unconditional_guidance_scale: 5.0
+          unconditional_guidance_label: [""]
+          ddim_steps: 50  # todo check these out for depth2img,
+          ddim_eta: 0.0   # todo check these out for depth2img,
+
+  trainer:
+    benchmark: True
+    val_check_interval: 5000000
+    num_sanity_val_steps: 0
+    accumulate_grad_batches: 1
diff --git a/v2_inference/v2-midas-inference.yaml b/v2_inference/v2-midas-inference.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f20c30f618b81091e31c2c4cf15325fa38638af4
--- /dev/null
+++ b/v2_inference/v2-midas-inference.yaml
@@ -0,0 +1,74 @@
+model:
+  base_learning_rate: 5.0e-07
+  target: ldm.models.diffusion.ddpm.LatentDepth2ImageDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: hybrid
+    scale_factor: 0.18215
+    monitor: val/loss_simple_ema
+    finetune_keys: null
+    use_ema: False
+
+    depth_stage_config:
+      target: ldm.modules.midas.api.MiDaSInference
+      params:
+        model_type: "dpt_hybrid"
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        image_size: 32 # unused
+        in_channels: 5
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_head_channels: 64 # need to fix for flash-attn
+        use_spatial_transformer: True
+        use_linear_in_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          #attn_type: "vanilla-xformers"
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+            - 1
+            - 2
+            - 4
+            - 4
+          num_res_blocks: 2
+          attn_resolutions: [ ]
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"
+
+
diff --git a/v2_inference/x4-upscaling.yaml b/v2_inference/x4-upscaling.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2db0964af699f86d1891c761710a2d53f59b842c
--- /dev/null
+++ b/v2_inference/x4-upscaling.yaml
@@ -0,0 +1,76 @@
+model:
+  base_learning_rate: 1.0e-04
+  target: ldm.models.diffusion.ddpm.LatentUpscaleDiffusion
+  params:
+    parameterization: "v"
+    low_scale_key: "lr"
+    linear_start: 0.0001
+    linear_end: 0.02
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: "jpg"
+    cond_stage_key: "txt"
+    image_size: 128
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: "hybrid-adm"
+    monitor: val/loss_simple_ema
+    scale_factor: 0.08333
+    use_ema: False
+
+    low_scale_config:
+      target: ldm.modules.diffusionmodules.upscaling.ImageConcatWithNoiseAugmentation
+      params:
+        noise_schedule_config: # image space
+          linear_start: 0.0001
+          linear_end: 0.02
+        max_noise_level: 350
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        use_checkpoint: True
+        num_classes: 1000  # timesteps for noise conditioning (here constant, just need one)
+        image_size: 128
+        in_channels: 7
+        out_channels: 4
+        model_channels: 256
+        attention_resolutions: [ 2,4,8]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 2, 4]
+        disable_self_attentions: [True, True, True, False]
+        disable_middle_self_attn: False
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 1024
+        legacy: False
+        use_linear_in_transformer: True
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        ddconfig:
+          # attn_type: "vanilla-xformers" this model needs efficient attention to be feasible on HR data, also the decoder seems to break in half precision (UNet is fine though)
+          double_z: True
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult: [ 1,2,4 ]  # num_down = len(ch_mult)-1
+          num_res_blocks: 2
+          attn_resolutions: [ ]
+          dropout: 0.0
+
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
+      params:
+        freeze: True
+        layer: "penultimate"
+