oh yeah these

Signed-off-by: Balazs Horvath <acsipont@gmail.com>

tf-xyfriend1-bleh

@@ -0,0 +1,96 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="friend-v1s2000"
+TRAINING_DIR="/home/kade/datasets/friend"
+OUTPUT_DIR="/home/kade/flux_output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+args=(
+    ## Model Paths
+    --pretrained_model_name_or_path ~/ComfyUI/models/unet/flux1-dev.safetensors
+    --clip_l ~/ComfyUI/models/clip/clip_l.safetensors
+    --t5xxl ~/ComfyUI/models/clip/t5xxl_fp16.safetensors
+    --ae ~/ComfyUI/models/vae/ae.safetensors
+    ## Network Arguments
+    # NOTE: Bad idea to train T5!
+    #--network_args
+    #    "train_t5xxl=True"
+    ## Timestep Sampling
+    --timestep_sampling shift
+    # `--discrete_flow_shift` is the discrete flow shift for the Euler Discrete Scheduler,
+    # default is 3.0 (same as SD3).
+    --discrete_flow_shift 3.1582
+    # `--model_prediction_type` is how to interpret and process the model prediction.
+    #   * `raw`: use as is, same as x-flux
+    #   * `additive`: add to noisy input
+    #   * `sigma_scaled`: apply sigma scaling, same as SD3
+    --model_prediction_type raw
+    --guidance_scale 1.0
+    # NOTE: In kohya's experiments,
+    # `--timestep_sampling shift --discrete_flow_shift 3.1582 --model_prediction_type raw --guidance_scale 1.0`
+    # (with the default `l2` `loss_type`) seems to work better.
+    #
+    # NOTE: The existing `--loss_type` option may be useful for FLUX.1 training. The default is `l2`.
+    #--loss_type l2
+    #
+    # Latents
+    --cache_latents_to_disk
+    --save_model_as safetensors
+    --sdpa
+    --persistent_data_loader_workers
+    --max_data_loader_n_workers 2
+    --seed 42
+    --max_train_steps=$STEPS
+    --gradient_checkpointing
+    --mixed_precision bf16
+    --optimizer_type=ClybW
+    --save_precision bf16
+    --network_module networks.lora_flux
+    --network_dim 4
+    --learning_rate 5e-4
+    --cache_text_encoder_outputs
+    --cache_text_encoder_outputs_to_disk
+    --fp8_base
+    --highvram
+    --dataset_config "$TRAINING_DIR/config.toml"
+    --output_dir $OUTPUT_DIR
+    --output_name $NAME
+    ## Sample Prompts
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_every_n_steps=20 
+    --sample_sampler="euler" 
+    --sample_at_first 
+    --save_every_n_steps=100
+)
+
+cd ~/source/repos/sd-scripts-sd3
+python "./flux_train_network.py" "${args[@]}"
+cd ~
+

tf-yyfriend

@@ -0,0 +1,96 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="yyfriend-v1s2000"
+TRAINING_DIR="/home/kade/datasets/yyfriend"
+OUTPUT_DIR="/home/kade/flux_output_dir/$NAME"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+args=(
+    ## Model Paths
+    --pretrained_model_name_or_path ~/ComfyUI/models/unet/flux1-dev.safetensors
+    --clip_l ~/ComfyUI/models/clip/clip_l.safetensors
+    --t5xxl ~/ComfyUI/models/clip/t5xxl_fp16.safetensors
+    --ae ~/ComfyUI/models/vae/ae.safetensors
+    ## Network Arguments
+    # NOTE: Bad idea to train T5!
+    #--network_args
+    #    "train_t5xxl=True"
+    ## Timestep Sampling
+    --timestep_sampling shift
+    # `--discrete_flow_shift` is the discrete flow shift for the Euler Discrete Scheduler,
+    # default is 3.0 (same as SD3).
+    --discrete_flow_shift 3.1582
+    # `--model_prediction_type` is how to interpret and process the model prediction.
+    #   * `raw`: use as is, same as x-flux
+    #   * `additive`: add to noisy input
+    #   * `sigma_scaled`: apply sigma scaling, same as SD3
+    --model_prediction_type raw
+    --guidance_scale 1.0
+    # NOTE: In kohya's experiments,
+    # `--timestep_sampling shift --discrete_flow_shift 3.1582 --model_prediction_type raw --guidance_scale 1.0`
+    # (with the default `l2` `loss_type`) seems to work better.
+    #
+    # NOTE: The existing `--loss_type` option may be useful for FLUX.1 training. The default is `l2`.
+    #--loss_type l2
+    #
+    # Latents
+    --cache_latents_to_disk
+    --save_model_as safetensors
+    --sdpa
+    --persistent_data_loader_workers
+    --max_data_loader_n_workers 2
+    --seed 42
+    --max_train_steps=$STEPS
+    --gradient_checkpointing
+    --mixed_precision bf16
+    --optimizer_type=ClybW
+    --save_precision bf16
+    --network_module networks.lora_flux
+    --network_dim 4
+    --learning_rate 5e-4
+    --cache_text_encoder_outputs
+    --cache_text_encoder_outputs_to_disk
+    --fp8_base
+    --highvram
+    --dataset_config "$TRAINING_DIR/config.toml"
+    --output_dir $OUTPUT_DIR
+    --output_name $NAME
+    ## Sample Prompts
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_every_n_steps=20 
+    --sample_sampler="euler" 
+    --sample_at_first 
+    --save_every_n_steps=100
+)
+
+cd ~/source/repos/sd-scripts-sd3
+python "./flux_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tf

@@ -0,0 +1,96 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="friend-v3s2000"
+TRAINING_DIR="/home/kade/datasets/friend"
+OUTPUT_DIR="/home/kade/flux_output_dir/$NAME"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+args=(
+    ## Model Paths
+    --pretrained_model_name_or_path ~/ComfyUI/models/unet/flux1-dev.safetensors
+    --clip_l ~/ComfyUI/models/clip/clip_l.safetensors
+    --t5xxl ~/ComfyUI/models/clip/t5xxl_fp16.safetensors
+    --ae ~/ComfyUI/models/vae/ae.safetensors
+    ## Network Arguments
+    # NOTE: Bad idea to train T5!
+    #--network_args
+    #    "train_t5xxl=True"
+    ## Timestep Sampling
+    --timestep_sampling shift
+    # `--discrete_flow_shift` is the discrete flow shift for the Euler Discrete Scheduler,
+    # default is 3.0 (same as SD3).
+    --discrete_flow_shift 3.1582
+    # `--model_prediction_type` is how to interpret and process the model prediction.
+    #   * `raw`: use as is, same as x-flux
+    #   * `additive`: add to noisy input
+    #   * `sigma_scaled`: apply sigma scaling, same as SD3
+    --model_prediction_type raw
+    --guidance_scale 1.0
+    # NOTE: In kohya's experiments,
+    # `--timestep_sampling shift --discrete_flow_shift 3.1582 --model_prediction_type raw --guidance_scale 1.0`
+    # (with the default `l2` `loss_type`) seems to work better.
+    #
+    # NOTE: The existing `--loss_type` option may be useful for FLUX.1 training. The default is `l2`.
+    #--loss_type l2
+    #
+    # Latents
+    --cache_latents_to_disk
+    --save_model_as safetensors
+    --sdpa
+    --persistent_data_loader_workers
+    --max_data_loader_n_workers 2
+    --seed 42
+    --max_train_steps=$STEPS
+    --gradient_checkpointing
+    --mixed_precision bf16
+    --optimizer_type=ClybW
+    --save_precision bf16
+    --network_module networks.lora_flux
+    --network_dim 4
+    --learning_rate 5e-4
+    --cache_text_encoder_outputs
+    --cache_text_encoder_outputs_to_disk
+    --fp8_base
+    --highvram
+    --dataset_config "$TRAINING_DIR/config.toml"
+    --output_dir $OUTPUT_DIR
+    --output_name $NAME
+    ## Sample Prompts
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_every_n_steps=20 
+    --sample_sampler="euler" 
+    --sample_at_first 
+    --save_every_n_steps=100
+)
+
+cd ~/source/repos/sd-scripts-sd3
+python "./flux_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-by_chunie-lokr

@@ -0,0 +1,125 @@
+#!/usr/bin/env zsh
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+conda activate sdscripts
+
+NAME="by_chunie-v3s1200"
+TRAINING_DIR="/home/kade/datasets/by_chunie"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark... 
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=32"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=3072
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=20
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-by_darkgem-lokr

@@ -0,0 +1,135 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="by_darkgem-v3s2400"
+TRAINING_DIR="/home/kade/datasets/by_darkgem"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=32"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --sample_at_first
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp-by_darkgem-lokr+factor

@@ -0,0 +1,135 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="by_darkgem-v4s2400"
+TRAINING_DIR="/home/kade/datasets/by_darkgem"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=16"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --sample_at_first
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp-by_darkgem-lokr+factor+lowerLR

@@ -0,0 +1,134 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="by_darkgem-v5s2400"
+TRAINING_DIR="/home/kade/datasets/by_darkgem"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=True"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=16"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=2
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --lr_warmup_steps=0
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --sample_at_first
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp-by_darkgem-lokr+factor+lowerLR_lowres

@@ -0,0 +1,134 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="by_darkgem-512b48-v5s800"
+TRAINING_DIR="/home/kade/datasets/by_darkgem"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="512,512"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=1024
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=16"
+               "use_tucker=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=48
+    #--gradient_accumulation_steps=1
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --lr_warmup_steps=0
+    --learning_rate=0.0003461
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --sample_at_first
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp-by_hamgas-lokr

@@ -0,0 +1,125 @@
+#!/usr/bin/env zsh
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+conda activate sdscripts
+
+NAME="by_hamgas-v3s1200"
+TRAINING_DIR="/home/kade/datasets/by_hamgas"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark... 
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=32"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=4096
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-by_jinxit-lokr

@@ -0,0 +1,137 @@
+#!/usr/bin/env zsh
+#
+# NOTE:
+# - Changed the compression factor of the LoKr.
+# 
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="by_jinxit-v3s6400"
+TRAINING_DIR="/home/kade/datasets/by_jinxit"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=16"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=200
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp-by_kousen

@@ -0,0 +1,116 @@
+#!/usr/bin/env zsh
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+conda activate sdscripts
+
+NAME="by_kousen-v1s400"
+TRAINING_DIR="/home/kade/datasets/by_kousen"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=256"
+               "conv_alpha=4"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=locon"
+               "dora_wd=False"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=400
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-by_kunaboto

@@ -0,0 +1,97 @@
+#!/usr/bin/env zsh
+
+conda activate sdscripts
+
+NAME="by_kunaboto-v1s600"
+TRAINING_DIR="/home/kade/training/by_kunaboto"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=256"
+               "conv_alpha=4"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=locon"
+               "dora_wd=False"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=600
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-by_latrans-lokr

@@ -0,0 +1,134 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="by_latrans-v3s1200"
+TRAINING_DIR="/home/kade/datasets/by_latrans"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=32"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp-by_melak

@@ -0,0 +1,116 @@
+#!/usr/bin/env zsh
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+conda activate sdscripts
+
+NAME="by_melak-v1s400"
+TRAINING_DIR="/home/kade/datasets/by_melak"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=256"
+               "conv_alpha=4"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=locon"
+               "dora_wd=False"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=400
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-by_randitawu

@@ -0,0 +1,116 @@
+#!/usr/bin/env zsh
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+conda activate sdscripts
+
+NAME="by_randitawu-v1s400"
+TRAINING_DIR="/home/kade/datasets/by_randitawu"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=64"
+               "conv_alpha=2"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=12
+    --gradient_accumulation_steps=4
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=400
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    #--multires_noise_iterations=12
+    #--multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-by_randitawu_normal

@@ -0,0 +1,116 @@
+#!/usr/bin/env zsh
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+conda activate sdscripts
+
+NAME="by_randitawu-v2s400"
+TRAINING_DIR="/home/kade/datasets/by_randitawu"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=256"
+               "conv_alpha=4"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=locon"
+               "dora_wd=False"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=400
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-by_wolfy-nail-lokr

@@ -0,0 +1,125 @@
+#!/usr/bin/env zsh
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+conda activate sdscripts
+
+NAME="by_wolfy-nail-v3s3000"
+TRAINING_DIR="/home/kade/datasets/by_wolfy-nail"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark... 
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=32"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=4096
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-curse_of_the_worgen

@@ -0,0 +1,118 @@
+#!/usr/bin/env zsh
+
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+
+conda activate sdscripts
+
+NAME="cotw-v1s400"
+TRAINING_DIR="/home/kade/datasets/curse_of_the_worgen"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=64"
+               "conv_alpha=2"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=12
+    --gradient_accumulation_steps=4
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=400
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    #--multires_noise_iterations=12
+    #--multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-curse_of_the_worgen_normal

@@ -0,0 +1,116 @@
+#!/usr/bin/env zsh
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+conda activate sdscripts
+
+NAME="cotw-v2s400"
+TRAINING_DIR="/home/kade/datasets/curse_of_the_worgen"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=256"
+               "conv_alpha=4"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=locon"
+               "dora_wd=False"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=400
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-maliketh

@@ -0,0 +1,134 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="maliketh-v2s2000"
+TRAINING_DIR="/home/kade/datasets/maliketh"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=0.0625
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=1"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=True"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=16"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=ClybW
+    --train_batch_size=14
+    #--gradient_accumulation_steps=1
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --lr_warmup_steps=0
+    --learning_rate=0.0001
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=1
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --sample_at_first
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp-maliketh-highalpha

@@ -0,0 +1,134 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="maliketh-v3s2000"
+TRAINING_DIR="/home/kade/datasets/maliketh"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=True"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=16"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=ClybW
+    --train_batch_size=14
+    #--gradient_accumulation_steps=1
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --lr_warmup_steps=0
+    --learning_rate=0.0001
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=1
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --sample_at_first
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp-nextgen

@@ -0,0 +1,134 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="by_darkgem-v5s2400"
+TRAINING_DIR="/home/kade/datasets/by_darkgem"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=True"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=16"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --lr_warmup_steps=0
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --sample_at_first
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp-nitw

@@ -0,0 +1,125 @@
+#!/usr/bin/env zsh
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+conda activate sdscripts
+
+NAME="nitw-v1s1200"
+TRAINING_DIR="/home/kade/datasets/night_in_the_woods"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark... 
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=32"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=1200
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=20
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-oldie

@@ -0,0 +1,97 @@
+#!/usr/bin/env zsh
+
+conda activate sdscripts
+
+NAME="bigfur-v1s2000"
+TRAINING_DIR="/home/kade/training/training_dir"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=256"
+               "conv_alpha=4"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=locon"
+               "dora_wd=False"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=ClybW
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=2000
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-promo_animals

@@ -0,0 +1,114 @@
+#!/usr/bin/env zsh
+
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+
+conda activate sdscripts
+
+NAME="promo_animals-v1s400"
+TRAINING_DIR="/home/kade/datasets/promo_animals/bullseye"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=256"
+               "conv_alpha=4"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=locon"
+               "dora_wd=False"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=12
+    --gradient_accumulation_steps=4
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=600
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-promo_animals-bullseye

@@ -0,0 +1,118 @@
+#!/usr/bin/env zsh
+
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+
+conda activate sdscripts
+
+NAME="bullseye-v5s400"
+TRAINING_DIR="/home/kade/datasets/promo_animals/bullseye"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=64"
+               "conv_alpha=2"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=12
+    --gradient_accumulation_steps=4
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=400
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    #--multires_noise_iterations=12
+    #--multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-promo_animals-bullseye-clybw

@@ -0,0 +1,118 @@
+#!/usr/bin/env zsh
+
+
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <<<
+
+
+conda activate sdscripts
+
+NAME="bullseye-v2s400"
+TRAINING_DIR="/home/kade/datasets/promo_animals/bullseye"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=32
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=8
+    --network_alpha=4
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=256"
+               "conv_alpha=4"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=locon"
+               "dora_wd=False"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=ClybW
+    --train_batch_size=12
+    --gradient_accumulation_steps=4
+    --max_grad_norm=1
+    --gradient_checkpointing
+    #--lr_warmup_steps=6
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=600
+    --learning_rate=0.0002
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2  
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~
+

training_scripts/tp-try_this-lokr

@@ -0,0 +1,134 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="by_hamgas-v3s1200"
+TRAINING_DIR="/home/kade/datasets/by_hamgas"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=32"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+python "./sdxl_train_network.py" "${args[@]}"
+cd ~

training_scripts/tp_EXPE_-by_darkgem-lokr

@@ -0,0 +1,136 @@
+#!/usr/bin/env zsh
+#
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
+        . "/home/kade/miniconda3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/kade/miniconda3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+# <<< conda initialize <
+
+conda activate sdscripts
+
+NAME="$(basename ${0})"
+TRAINING_DIR="/home/kade/datasets/by_darkgem"
+OUTPUT_DIR="/home/kade/output_dir"
+
+# Extract the number of steps from the NAME
+STEPS=$(echo $NAME | grep -oE '[0-9]+$')
+
+# If no number is found at the end of NAME, set a default value
+if [ -z "$STEPS" ]; then
+    STEPS=4096
+    echo "No step count found in $NAME. Using default value of \e[35m$STEPS\e[0m"
+else
+    echo "Extracted \e[35m$STEPS\e[0m steps from $NAME"
+fi
+
+# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
+# --min_snr_gamma=1
+args=(
+    # ⚠️  TODO: Benchmark...
+    --debiased_estimation_loss
+    # ⚠️  TODO: What does this do? Does it even work?
+    --max_token_length=225
+    # Keep Tokens
+    --keep_tokens=1
+    --keep_tokens_separator="|||"
+    # Model
+    --pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
+    # Output, logging
+    --output_dir="$OUTPUT_DIR/$NAME"
+    --output_name="$NAME"
+    --log_prefix="$NAME-"
+    --log_with=tensorboard
+    --logging_dir="$OUTPUT_DIR/logs"
+    --seed=1728871242
+    # Dataset
+    --train_data_dir="$TRAINING_DIR"
+    --dataset_repeats=1
+    --resolution="1024,1024"
+    --enable_bucket
+    --bucket_reso_steps=64
+    --min_bucket_reso=256
+    --max_bucket_reso=2048
+    --flip_aug
+    --shuffle_caption
+    --cache_latents
+    --cache_latents_to_disk
+    --max_data_loader_n_workers=8
+    --persistent_data_loader_workers
+    # Network config
+    --network_dim=100000
+    # ⚠️  TODO: Plot
+    --network_alpha=64
+    --network_module="lycoris.kohya"
+    --network_args
+               "preset=full"
+               "conv_dim=100000"
+               "decompose_both=False"
+               "conv_alpha=64"
+               "rank_dropout=0"
+               "module_dropout=0"
+               "use_tucker=False"
+               "use_scalar=False"
+               "rank_dropout_scale=False"
+               "algo=lokr"
+               "bypass_mode=False"
+               "factor=32"
+               "use_cp=True"
+               "dora_wd=True"
+               "train_norm=False"
+    --network_dropout=0
+    # Optimizer config
+    --optimizer_type=FCompass
+    --train_batch_size=8
+    --gradient_accumulation_steps=6
+    --max_grad_norm=1
+    --gradient_checkpointing
+    --lr_warmup_steps=0
+    #--scale_weight_norms=1
+    # LR Scheduling
+    --max_train_steps=$STEPS
+    --learning_rate=0.0005
+    --unet_lr=0.0002
+    --text_encoder_lr=0.0001
+    --lr_scheduler="cosine"
+    --lr_scheduler_args="num_cycles=0.375"
+    # Noise
+    --multires_noise_iterations=12
+    --multires_noise_discount=0.4
+    #--min_snr_gamma=1
+    # Optimization, details
+    --no_half_vae
+    --sdpa
+    --mixed_precision="bf16"
+    # Saving
+    --save_model_as="safetensors"
+    --save_precision="fp16"
+    --save_every_n_steps=100
+    # Saving States
+    #--save_state
+    # Either resume from a saved state
+    #--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
+    #--skip_until_initial_step
+    # Or from a checkpoint
+    #--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
+    #--initial_step=120
+    # Sampling
+    --sample_every_n_steps=100
+    --sample_prompts="$TRAINING_DIR/sample-prompts.txt"
+    --sample_sampler="euler_a"
+    --sample_at_first
+    --caption_extension=".txt"
+)
+cd ~/source/repos/sd-scripts
+#accelerate launch --num_cpu_threads_per_process=2
+echo "Running: python \"./sdxl_train_network.py\" \"${args[@]}\" \"$@\""
+python "./sdxl_train_network.py" "${args[@]}" "$@"
+cd ~