File size: 4,016 Bytes
68fb7d8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 |
#!/usr/bin/env zsh
#
# >>> conda initialize >>>
# !! Contents within this block are managed by 'conda init' !!
__conda_setup="$('/home/kade/miniconda3/bin/conda' 'shell.zsh' 'hook' 2> /dev/null)"
if [ $? -eq 0 ]; then
eval "$__conda_setup"
else
if [ -f "/home/kade/miniconda3/etc/profile.d/conda.sh" ]; then
. "/home/kade/miniconda3/etc/profile.d/conda.sh"
else
export PATH="/home/kade/miniconda3/bin:$PATH"
fi
fi
unset __conda_setup
# <<< conda initialize <
conda activate sdscripts
NAME="by_latrans-v3s1200"
TRAINING_DIR="/home/kade/datasets/by_latrans"
OUTPUT_DIR="/home/kade/output_dir"
# Extract the number of steps from the NAME
STEPS=$(echo $NAME | grep -oE '[0-9]+$')
# If no number is found at the end of NAME, set a default value
if [ -z "$STEPS" ]; then
STEPS=4096
echo "No step count found in NAME. Using default value of \e[35m$STEPS\e[0m"
else
echo "Extracted \e[35m$STEPS\e[0m steps from NAME"
fi
# alpha=1 @ dim=16 is the same lr than alpha=4 @ dim=256
# --min_snr_gamma=1
args=(
# ⚠️ TODO: Benchmark...
--debiased_estimation_loss
# ⚠️ TODO: What does this do? Does it even work?
--max_token_length=225
# Keep Tokens
--keep_tokens=1
--keep_tokens_separator="|||"
# Model
--pretrained_model_name_or_path=/home/kade/ComfyUI/models/checkpoints/ponyDiffusionV6XL_v6StartWithThisOne.safetensors
# Output, logging
--output_dir="$OUTPUT_DIR/$NAME"
--output_name="$NAME"
--log_prefix="$NAME-"
--log_with=tensorboard
--logging_dir="$OUTPUT_DIR/logs"
--seed=1728871242
# Dataset
--train_data_dir="$TRAINING_DIR"
--dataset_repeats=1
--resolution="1024,1024"
--enable_bucket
--bucket_reso_steps=64
--min_bucket_reso=256
--max_bucket_reso=2048
--flip_aug
--shuffle_caption
--cache_latents
--cache_latents_to_disk
--max_data_loader_n_workers=8
--persistent_data_loader_workers
# Network config
--network_dim=100000
# ⚠️ TODO: Plot
--network_alpha=64
--network_module="lycoris.kohya"
--network_args
"preset=full"
"conv_dim=100000"
"decompose_both=False"
"conv_alpha=64"
"rank_dropout=0"
"module_dropout=0"
"use_tucker=False"
"use_scalar=False"
"rank_dropout_scale=False"
"algo=lokr"
"bypass_mode=False"
"factor=32"
"use_cp=True"
"dora_wd=True"
"train_norm=False"
--network_dropout=0
# Optimizer config
--optimizer_type=FCompass
--train_batch_size=8
--gradient_accumulation_steps=6
--max_grad_norm=1
--gradient_checkpointing
--lr_warmup_steps=0
#--scale_weight_norms=1
# LR Scheduling
--max_train_steps=$STEPS
--learning_rate=0.0005
--unet_lr=0.0002
--text_encoder_lr=0.0001
--lr_scheduler="cosine"
--lr_scheduler_args="num_cycles=0.375"
# Noise
--multires_noise_iterations=12
--multires_noise_discount=0.4
#--min_snr_gamma=1
# Optimization, details
--no_half_vae
--sdpa
--mixed_precision="bf16"
# Saving
--save_model_as="safetensors"
--save_precision="fp16"
--save_every_n_steps=100
# Saving States
#--save_state
# Either resume from a saved state
#--resume="$OUTPUT_DIR/wolflink-vfucks400" # Resume from saved state
#--skip_until_initial_step
# Or from a checkpoint
#--network_weights="$OUTPUT_DIR/wolflink-vfucks400/wolflink-vfucks400-step00000120.safetensors" # Resume from checkpoint (not needed with state, i think)
#--initial_step=120
# Sampling
--sample_every_n_steps=100
--sample_prompts="$TRAINING_DIR/sample-prompts.txt"
--sample_sampler="euler_a"
--caption_extension=".txt"
)
cd ~/source/repos/sd-scripts
#accelerate launch --num_cpu_threads_per_process=2
python "./sdxl_train_network.py" "${args[@]}"
cd ~
|