F go

README.md

@@ -1,19 +1,16 @@
 ---
-language:
-- en
-thumbnail: ''
+language: 
+  - en
+thumbnail: ""
 tags:
 - controlnet
 - laion
 - face
 - mediapipe
-- image-to-image
-license: openrail
-base_model: stabilityai/stable-diffusion-2-1-base
+license: "openrail"
 datasets:
 - LAION-Face
 - LAION
-pipeline_tag: image-to-image
 ---
 
 # ControlNet LAION Face Dataset
@@ -107,58 +104,12 @@ python ./train_laion_face_sd15.py
 We have provided `gradio_face2image.py`.  Update the following two lines to point them to your trained model.
 
 ```
-model = create_model('./models/cldm_v21.yaml').cpu()  # If you fine-tune on SD2.1 base, this does not need to change.
+model = create_model('./models/cldm_v21.yaml').cpu()  # If you fine-tuned on SD2.1 base, this does not need to change.
 model.load_state_dict(load_state_dict('./models/control_sd21_openpose.pth', location='cuda'))
 ```
 
 The model has some limitations: while it is empirically better at tracking gaze and mouth poses than previous attempts, it may still ignore controls.  Adding details to the prompt like, "looking right" can abate bad behavior. 
 
-## 🧨 Diffusers
-
-It is recommended to use the checkpoint with [Stable Diffusion 2.1 - Base](stabilityai/stable-diffusion-2-1-base) as the checkpoint has been trained on it.
-Experimentally, the checkpoint can be used with other diffusion models such as dreamboothed stable diffusion.
-
-To use with Stable Diffusion 1.5, insert `subfolder="diffusion_sd15"` into the from_pretrained arguments.  A v1.5 half-precision variant is provided but untested.
-
-1. Install `diffusers` and related packages:
-```
-$ pip install diffusers transformers accelerate
-```
-
-2. Run code:
-```py
-from PIL import Image
-import numpy as np
-import torch
-from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
-from diffusers.utils import load_image
-
-image = load_image(
-    "https://huggingface.co/CrucibleAI/ControlNetMediaPipeFace/resolve/main/samples_laion_face_dataset/family_annotation.png"
-)
-
-# Stable Diffusion 2.1-base:
-controlnet = ControlNetModel.from_pretrained("CrucibleAI/ControlNetMediaPipeFace", torch_dtype=torch.float16, variant="fp16")
-pipe = StableDiffusionControlNetPipeline.from_pretrained(
-	"stabilityai/stable-diffusion-2-1-base", controlnet=controlnet, safety_checker=None, torch_dtype=torch.float16
-)
-# OR
-# Stable Diffusion 1.5:
-controlnet = ControlNetModel.from_pretrained("CrucibleAI/ControlNetMediaPipeFace", subfolder="diffusion_sd15")
-pipe = StableDiffusionControlNetPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", controlnet=controlnet, safety_checker=None)
-
-pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
-
-# Remove if you do not have xformers installed
-# see https://huggingface.co/docs/diffusers/v0.13.0/en/optimization/xformers#installing-xformers
-# for installation instructions
-pipe.enable_xformers_memory_efficient_attention()
-pipe.enable_model_cpu_offload()
-
-image = pipe("a happy family at a dentist advertisement", image=image, num_inference_steps=30).images[0]
-image.save('./images.png')
-```
-
 
 # License:
 
@@ -209,4 +160,4 @@ Sample images for this document were obtained from Unsplash and are CC0.
 }
 ```
 
-This project was made possible by Crucible AI.
+This project was made possible by Crucible AI.

config.json

@@ -1,47 +0,0 @@
-{
-  "_class_name": "ControlNetModel",
-  "_diffusers_version": "0.15.0.dev0",
-  "_name_or_path": "/home/patrick_huggingface_co/temp_control",
-  "act_fn": "silu",
-  "attention_head_dim": [
-    5,
-    10,
-    20,
-    20
-  ],
-  "block_out_channels": [
-    320,
-    640,
-    1280,
-    1280
-  ],
-  "class_embed_type": null,
-  "conditioning_embedding_out_channels": [
-    16,
-    32,
-    96,
-    256
-  ],
-  "controlnet_conditioning_channel_order": "rgb",
-  "cross_attention_dim": 1024,
-  "down_block_types": [
-    "CrossAttnDownBlock2D",
-    "CrossAttnDownBlock2D",
-    "CrossAttnDownBlock2D",
-    "DownBlock2D"
-  ],
-  "downsample_padding": 1,
-  "flip_sin_to_cos": true,
-  "freq_shift": 0,
-  "in_channels": 4,
-  "layers_per_block": 2,
-  "mid_block_scale_factor": 1,
-  "norm_eps": 1e-05,
-  "norm_num_groups": 32,
-  "num_class_embeds": null,
-  "only_cross_attention": false,
-  "projection_class_embeddings_input_dim": null,
-  "resnet_time_scale_shift": "default",
-  "upcast_attention": false,
-  "use_linear_projection": true
-}

control_v2p_sd15_mediapipe_face.full.ckpt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a2a71953d7372d5585899b44693a7532ebbf80c091108ae2b8987ca93cc2dac2
-size 8601300183

control_v2p_sd15_mediapipe_face.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2f2ccead3a8c0b9fbf9cad7b8eaa29834983ced916c766a92fb84db34ff29e43
-size 1445239863

control_v2p_sd15_mediapipe_face.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5be501156709895f0b14a7ec76faae7cf0a105f76895252a2c69db541629628f
-size 1445154814

control_v2p_sd15_mediapipe_face.yaml

@@ -1,79 +0,0 @@
-model:
-  target: cldm.cldm.ControlLDM
-  params:
-    linear_start: 0.00085
-    linear_end: 0.0120
-    num_timesteps_cond: 1
-    log_every_t: 200
-    timesteps: 1000
-    first_stage_key: "jpg"
-    cond_stage_key: "txt"
-    control_key: "hint"
-    image_size: 64
-    channels: 4
-    cond_stage_trainable: false
-    conditioning_key: crossattn
-    monitor: val/loss_simple_ema
-    scale_factor: 0.18215
-    use_ema: False
-    only_mid_control: False
-
-    control_stage_config:
-      target: cldm.cldm.ControlNet
-      params:
-        image_size: 32 # unused
-        in_channels: 4
-        hint_channels: 3
-        model_channels: 320
-        attention_resolutions: [ 4, 2, 1 ]
-        num_res_blocks: 2
-        channel_mult: [ 1, 2, 4, 4 ]
-        num_heads: 8
-        use_spatial_transformer: True
-        transformer_depth: 1
-        context_dim: 768
-        use_checkpoint: True
-        legacy: False
-
-    unet_config:
-      target: cldm.cldm.ControlledUnetModel
-      params:
-        image_size: 32 # unused
-        in_channels: 4
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions: [ 4, 2, 1 ]
-        num_res_blocks: 2
-        channel_mult: [ 1, 2, 4, 4 ]
-        num_heads: 8
-        use_spatial_transformer: True
-        transformer_depth: 1
-        context_dim: 768
-        use_checkpoint: True
-        legacy: False
-
-    first_stage_config:
-      target: ldm.models.autoencoder.AutoencoderKL
-      params:
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          double_z: true
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult:
-          - 1
-          - 2
-          - 4
-          - 4
-          num_res_blocks: 2
-          attn_resolutions: []
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity
-
-    cond_stage_config:
-      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder

diffusion_pytorch_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:36dcd318d499df44b35432599a1b70f598e7bb42b479e4e67d4adf7b7e87e87d
-size 1457051321

diffusion_pytorch_model.fp16.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7f70c38860e0d1fcd0f5ed38bc34e61c7337b9001bed57f7bff6eba6471406f0
-size 728596455

diffusion_pytorch_model.fp16.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:02b3a8e04154b4c3d11f5210217f0dbf3fac8612d62d015cd059f2b9fe4c3364
-size 728496846

diffusion_pytorch_model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a683e98e2427fd6242edc9af6620708f2f8fc84bfc049fafe549e350f8d42d73
-size 1456953564

diffusion_sd15/config.json

@@ -1,42 +0,0 @@
-{
-  "_class_name": "ControlNetModel",
-  "_diffusers_version": "0.15.0.dev0",
-  "_name_or_path": "/home/josephcatrambone/ControlNet/models",
-  "act_fn": "silu",
-  "attention_head_dim": 8,
-  "block_out_channels": [
-    320,
-    640,
-    1280,
-    1280
-  ],
-  "class_embed_type": null,
-  "conditioning_embedding_out_channels": [
-    16,
-    32,
-    96,
-    256
-  ],
-  "controlnet_conditioning_channel_order": "rgb",
-  "cross_attention_dim": 768,
-  "down_block_types": [
-    "CrossAttnDownBlock2D",
-    "CrossAttnDownBlock2D",
-    "CrossAttnDownBlock2D",
-    "DownBlock2D"
-  ],
-  "downsample_padding": 1,
-  "flip_sin_to_cos": true,
-  "freq_shift": 0,
-  "in_channels": 4,
-  "layers_per_block": 2,
-  "mid_block_scale_factor": 1,
-  "norm_eps": 1e-05,
-  "norm_num_groups": 32,
-  "num_class_embeds": null,
-  "only_cross_attention": false,
-  "projection_class_embeddings_input_dim": null,
-  "resnet_time_scale_shift": "default",
-  "upcast_attention": null,
-  "use_linear_projection": false
-}

diffusion_sd15/diffusion_pytorch_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f63de389f776b75bb11f10487a187573aea84f9a51debd08f314bd084e7fb362
-size 1445254969

diffusion_sd15/diffusion_pytorch_model.fp16.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0c37b3dd41e956160909129b50f84fd938116550727b491192cbdbe6f896cd7b
-size 722696633

diffusion_sd15/diffusion_pytorch_model.fp16.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9fb50465b4fd7e15f0dc7df8031767e57309cfda2917082485bcf6c11bedb540
-size 722598642

gradio_face2image.py

@@ -13,8 +13,8 @@ from laion_face_common import generate_annotation
 from share import *
 
 
-model = create_model('./control_v2p_sd21_mediapipe_face.yaml').cpu()
-model.load_state_dict(load_state_dict('./control_v2p_sd21_mediapipe_face.full.ckpt', location='cuda'))
+model = create_model('./models/cldm_v21.yaml').cpu()
+model.load_state_dict(load_state_dict('./models/controlnet_face_condition_epoch_4_0percent.ckpt', location='cuda'))
 model = model.cuda()
 ddim_sampler = DDIMSampler(model)  # ControlNet _only_ works with DDIM.