| The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`. | |
| Running 1 job | |
| 0it [00:00, ?it/s] 0it [00:00, ?it/s] | |
| /usr/local/lib/python3.10/dist-packages/controlnet_aux/mediapipe_face/mediapipe_face_common.py:7: UserWarning: The module 'mediapipe' is not installed. The package will have limited functionality. Please install it using the command: pip install 'mediapipe' | |
| warnings.warn( | |
| /usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_5m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_5m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected. | |
| return register_model(fn_wrapper) | |
| /usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_11m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_11m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected. | |
| return register_model(fn_wrapper) | |
| /usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected. | |
| return register_model(fn_wrapper) | |
| /usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_384 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_384. This is because the name being registered conflicts with an existing name. Please check if this is not expected. | |
| return register_model(fn_wrapper) | |
| /usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_512 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_512. This is because the name being registered conflicts with an existing name. Please check if this is not expected. | |
| return register_model(fn_wrapper) | |
| You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers | |
| { | |
| "type": "sd_trainer", | |
| "training_folder": "output", | |
| "device": "cuda:0", | |
| "network": { | |
| "type": "lora", | |
| "linear": 16, | |
| "linear_alpha": 16 | |
| }, | |
| "save": { | |
| "dtype": "float16", | |
| "save_every": 250, | |
| "max_step_saves_to_keep": 4, | |
| "push_to_hub": false | |
| }, | |
| "datasets": [ | |
| { | |
| "folder_path": "/workspace/ai-toolkit/images", | |
| "caption_ext": "txt", | |
| "caption_dropout_rate": 0.05, | |
| "shuffle_tokens": false, | |
| "cache_latents_to_disk": true, | |
| "resolution": [ | |
| 512, | |
| 768, | |
| 1024 | |
| ] | |
| } | |
| ], | |
| "train": { | |
| "batch_size": 1, | |
| "steps": 2000, | |
| "gradient_accumulation_steps": 1, | |
| "train_unet": true, | |
| "train_text_encoder": false, | |
| "gradient_checkpointing": true, | |
| "noise_scheduler": "flowmatch", | |
| "optimizer": "adamw8bit", | |
| "lr": 0.0001, | |
| "ema_config": { | |
| "use_ema": true, | |
| "ema_decay": 0.99 | |
| }, | |
| "dtype": "bf16" | |
| }, | |
| "model": { | |
| "name_or_path": "black-forest-labs/FLUX.1-dev", | |
| "is_flux": true, | |
| "quantize": true | |
| }, | |
| "sample": { | |
| "sampler": "flowmatch", | |
| "sample_every": 250, | |
| "width": 1024, | |
| "height": 1024, | |
| "prompts": [ | |
| "woman with red hair, playing chess at the park, bomb going off in the background", | |
| "a woman holding a coffee cup, in a beanie, sitting at a cafe", | |
| "a horse is a DJ at a night club, fish eye lens, smoke machine, lazer lights, holding a martini", | |
| "a man showing off his cool new t shirt at the beach, a shark is jumping out of the water in the background", | |
| "a bear building a log cabin in the snow covered mountains", | |
| "woman playing the guitar, on stage, singing a song, laser lights, punk rocker", | |
| "hipster man with a beard, building a chair, in a wood shop", | |
| "photo of a man, white background, medium shot, modeling clothing, studio lighting, white backdrop", | |
| "a man holding a sign that says, 'this is a sign'", | |
| "a bulldog, in a post apocalyptic world, with a shotgun, in a leather jacket, in a desert, with a motorcycle" | |
| ], | |
| "neg": "", | |
| "seed": 42, | |
| "walk_seed": true, | |
| "guidance_scale": 4, | |
| "sample_steps": 20 | |
| } | |
| } | |
| Using EMA | |
| ############################################# | |
| # Running job: my_first_flux_lora_v1 | |
| ############################################# | |
| Running 1 process | |
| Loading Flux model | |
| Loading transformer | |
| Quantizing transformer | |
| Loading vae | |
| Loading t5 | |
| Downloading shards: 0%| | 0/2 [00:00<?, ?it/s] Downloading shards: 50%|█████ | 1/2 [00:04<00:04, 4.38s/it] Downloading shards: 100%|██████████| 2/2 [00:08<00:00, 4.22s/it] Downloading shards: 100%|██████████| 2/2 [00:08<00:00, 4.25s/it] | |
| Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s] Loading checkpoint shards: 50%|█████ | 1/2 [00:00<00:00, 8.42it/s] Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00, 9.07it/s] Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00, 8.96it/s] | |
| Quantizing T5 | |
| Loading clip | |
| making pipe | |
| preparing | |
| create LoRA network. base dim (rank): 16, alpha: 16 | |
| neuron dropout: p=None, rank dropout: p=None, module dropout: p=None | |
| create LoRA for Text Encoder: 0 modules. | |
| create LoRA for U-Net: 494 modules. | |
| enable LoRA for U-Net | |
| Error running job: [Errno 2] No such file or directory: '/workspace/ai-toolkit/images' | |
| ======================================== | |
| Result: | |
| - 0 completed jobs | |
| - 1 failure | |
| ======================================== | |
| Traceback (most recent call last): | |
| File "/workspace/ai-toolkit/run.py", line 90, in <module> | |
| main() | |
| File "/workspace/ai-toolkit/run.py", line 86, in main | |
| raise e | |
| File "/workspace/ai-toolkit/run.py", line 78, in main | |
| job.run() | |
| File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run | |
| process.run() | |
| File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 1567, in run | |
| self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd) | |
| File "/workspace/ai-toolkit/toolkit/data_loader.py", line 571, in get_dataloader_from_datasets | |
| dataset = AiToolkitDataset(config, batch_size=batch_size, sd=sd) | |
| File "/workspace/ai-toolkit/toolkit/data_loader.py", line 411, in __init__ | |
| with open(self.dataset_path, 'r') as f: | |
| FileNotFoundError: [Errno 2] No such file or directory: '/workspace/ai-toolkit/images' | |