python3 -m accelerate.commands.launch \
    --multi_gpu \
    --num_processes=2 \
    --num_cpu_threads_per_process=16 sdxl_train_network.py \
    --enable_bucket \
    --optimizer_type Lion8bit \
    --save_state \
    --network_args \
        conv_dim=10000 \
        conv_alpha=10000 \
        algo=lokr \
        dropout=0 \
        factor=8 \
        train_norm=True \
        preset=./sdxl-base.toml \
    --noise_offset 0.05 \
    --min_snr_gamma 5 \
    --pretrained_model_name_or_path=sd-models/kohaku-xl-delta-pre2.safetensors \
    --train_data_dir=datasets/danbooru/kxl-delta \
    --output_dir=output \
    --logging_dir=./logs \
    --log_prefix=kohaku-xl-delta-lyco \
    --resolution=1024,1024 \
    --network_module=lycoris.kohya \
    --max_train_epochs=1 \
    --save_every_n_steps=2000 \
    --unet_lr=4e-5 \
    --text_encoder_lr=1e-5 \
    --lr_scheduler=constant_with_warmup \
    --lr_warmup_steps=100 \
    --lr_scheduler_num_cycles=1 \
    --network_dim=10000 \
    --network_alpha=10000 \
    --output_name=kohaku-xl-delta-lyco \
    --train_batch_size=4 \
    --gradient_accumulation_steps=16 \
    --save_every_n_epochs=1 \
    --mixed_precision=fp16 \
    --save_precision=fp16 \
    --full_fp16 \
    --seed=1337 \
    --prior_loss_weight=1 \
    --max_token_length=225 \
    --caption_extension=.txt \
    --save_model_as=safetensors \
    --min_bucket_reso=256 \
    --max_bucket_reso=4096 \
    --keep_tokens=0 \
    --xformers \
    --gradient_checkpointing \
    --shuffle_caption \
    --caption_tag_dropout_rate=0.1 \
    --keep_tokens_separator="|||" \
    --optimizer_args weight_decay=0.1 betas=0.9,0.95 \
    --ddp_gradient_as_bucket_view \
    --ddp_timeout 10080