Upload 5 files

README (4).md

@@ -0,0 +1,13 @@
+---
+title: PuLID-FLUX
+emoji: 🤗
+colorFrom: blue
+colorTo: indigo
+sdk: gradio
+sdk_version: 4.44.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app (8).py

@@ -0,0 +1,316 @@
+import spaces
+import time
+import os
+
+import gradio as gr
+import torch
+from einops import rearrange
+from PIL import Image
+from transformers import pipeline
+
+from flux.cli import SamplingOptions
+from flux.sampling import denoise, get_noise, get_schedule, prepare, unpack
+from flux.util import load_ae, load_clip, load_flow_model, load_t5
+from pulid.pipeline_flux import PuLIDPipeline
+from pulid.utils import resize_numpy_image_long
+
+NSFW_THRESHOLD = 0.85
+
+def get_models(name: str, device: torch.device, offload: bool):
+    t5 = load_t5(device, max_length=128)
+    clip = load_clip(device)
+    model = load_flow_model(name, device="cpu" if offload else device)
+    model.eval()
+    ae = load_ae(name, device="cpu" if offload else device)
+    nsfw_classifier = pipeline("image-classification", model="Falconsai/nsfw_image_detection", device=device)
+    return model, ae, t5, clip, nsfw_classifier
+
+
+class FluxGenerator:
+    def __init__(self):
+        self.device = torch.device('cuda')
+        self.offload = False
+        self.model_name = 'flux-dev'
+        self.model, self.ae, self.t5, self.clip, self.nsfw_classifier = get_models(
+            self.model_name,
+            device=self.device,
+            offload=self.offload,
+        )
+        self.pulid_model = PuLIDPipeline(self.model, 'cuda', weight_dtype=torch.bfloat16)
+        self.pulid_model.load_pretrain()
+
+
+flux_generator = FluxGenerator()
+
+
+@spaces.GPU
+@torch.inference_mode()
+def generate_image(
+        prompt,
+        id_image,
+        start_step,
+        guidance,
+        seed,
+        true_cfg,
+        width=896,
+        height=1152,
+        num_steps=20,
+        id_weight=1.0,
+        neg_prompt="bad quality, worst quality, text, signature, watermark, extra limbs",
+        timestep_to_start_cfg=1,
+        max_sequence_length=128,
+):
+    flux_generator.t5.max_length = max_sequence_length
+
+    seed = int(seed)
+    if seed == -1:
+        seed = None
+
+    opts = SamplingOptions(
+        prompt=prompt,
+        width=width,
+        height=height,
+        num_steps=num_steps,
+        guidance=guidance,
+        seed=seed,
+    )
+
+    if opts.seed is None:
+        opts.seed = torch.Generator(device="cpu").seed()
+    print(f"Generating '{opts.prompt}' with seed {opts.seed}")
+    t0 = time.perf_counter()
+
+    use_true_cfg = abs(true_cfg - 1.0) > 1e-2
+
+    if id_image is not None:
+        id_image = resize_numpy_image_long(id_image, 1024)
+        id_embeddings, uncond_id_embeddings = flux_generator.pulid_model.get_id_embedding(id_image, cal_uncond=use_true_cfg)
+    else:
+        id_embeddings = None
+        uncond_id_embeddings = None
+
+
+    # prepare input
+    x = get_noise(
+        1,
+        opts.height,
+        opts.width,
+        device=flux_generator.device,
+        dtype=torch.bfloat16,
+        seed=opts.seed,
+    )
+    timesteps = get_schedule(
+        opts.num_steps,
+        x.shape[-1] * x.shape[-2] // 4,
+        shift=True,
+    )
+
+    if flux_generator.offload:
+        flux_generator.t5, flux_generator.clip = flux_generator.t5.to(flux_generator.device), flux_generator.clip.to(flux_generator.device)
+    inp = prepare(t5=flux_generator.t5, clip=flux_generator.clip, img=x, prompt=opts.prompt)
+    inp_neg = prepare(t5=flux_generator.t5, clip=flux_generator.clip, img=x, prompt=neg_prompt) if use_true_cfg else None
+
+    # offload TEs to CPU, load model to gpu
+    if flux_generator.offload:
+        flux_generator.t5, flux_generator.clip = flux_generator.t5.cpu(), flux_generator.clip.cpu()
+        torch.cuda.empty_cache()
+        flux_generator.model = flux_generator.model.to(flux_generator.device)
+
+    # denoise initial noise
+    x = denoise(
+        flux_generator.model, **inp, timesteps=timesteps, guidance=opts.guidance, id=id_embeddings, id_weight=id_weight,
+        start_step=start_step, uncond_id=uncond_id_embeddings, true_cfg=true_cfg,
+        timestep_to_start_cfg=timestep_to_start_cfg,
+        neg_txt=inp_neg["txt"] if use_true_cfg else None,
+        neg_txt_ids=inp_neg["txt_ids"] if use_true_cfg else None,
+        neg_vec=inp_neg["vec"] if use_true_cfg else None,
+    )
+
+    # offload model, load autoencoder to gpu
+    if flux_generator.offload:
+        flux_generator.model.cpu()
+        torch.cuda.empty_cache()
+        flux_generator.ae.decoder.to(x.device)
+
+    # decode latents to pixel space
+    x = unpack(x.float(), opts.height, opts.width)
+    with torch.autocast(device_type=flux_generator.device.type, dtype=torch.bfloat16):
+        x = flux_generator.ae.decode(x)
+
+    if flux_generator.offload:
+        flux_generator.ae.decoder.cpu()
+        torch.cuda.empty_cache()
+
+    t1 = time.perf_counter()
+
+    print(f"Done in {t1 - t0:.1f}s.")
+    # bring into PIL format
+    x = x.clamp(-1, 1)
+    # x = embed_watermark(x.float())
+    x = rearrange(x[0], "c h w -> h w c")
+
+    img = Image.fromarray((127.5 * (x + 1.0)).cpu().byte().numpy())
+    nsfw_score = [x["score"] for x in flux_generator.nsfw_classifier(img) if x["label"] == "nsfw"][0]
+    if nsfw_score < NSFW_THRESHOLD:
+        return img, str(opts.seed), flux_generator.pulid_model.debug_img_list
+    else:
+        return (None, f"Your generated image may contain NSFW (with nsfw_score: {nsfw_score}) content",
+                flux_generator.pulid_model.debug_img_list)
+
+_HEADER_ = '''
+<div style="text-align: center; max-width: 650px; margin: 0 auto;">
+    <h1 style="font-size: 2.5rem; font-weight: 700; margin-bottom: 1rem; display: contents;">PuLID for FLUX</h1>
+    <p style="font-size: 1rem; margin-bottom: 1.5rem;">Paper: <a href='https://arxiv.org/abs/2404.16022' target='_blank'>PuLID: Pure and Lightning ID Customization via Contrastive Alignment</a> | Codes: <a href='https://github.com/ToTheBeginning/PuLID' target='_blank'>GitHub</a></p>
+</div>
+
+❗️❗️❗️**Tips:**
+
+- `timestep to start inserting ID:` The smaller the value, the higher the fidelity, but the lower the editability; the higher the value, the lower the fidelity, but the higher the editability. **The recommended range for this value is between 0 and 4**. For photorealistic scenes, we recommend using 4; for stylized scenes, we recommend using 0-1. If you are not satisfied with the similarity, you can lower this value; conversely, if you are not satisfied with the editability, you can increase this value.
+- `true CFG scale:` In most scenarios, it is recommended to use a fake CFG, i.e., setting the true CFG scale to 1, and just adjusting the guidance scale. This is also more efficiency. However, in a few cases, utilizing a true CFG can yield better results. For more detaileds, please refer to the [doc](https://github.com/ToTheBeginning/PuLID/blob/main/docs/pulid_for_flux.md#useful-tips).
+- `Learn more about the model:` please refer to the <a href='https://github.com/ToTheBeginning/PuLID/blob/main/docs/pulid_for_flux.md' target='_blank'>github doc</a> for more details and info about the model, we provide the detail explanation about the above two parameters in the doc.
+- `Examples:` we provide some examples (we have cached them, so just click them to see what the model can do) in the bottom, you can try these example prompts first
+'''  # noqa E501
+
+_CITE_ = r"""
+If PuLID is helpful, please help to ⭐ the <a href='https://github.com/ToTheBeginning/PuLID' target='_blank'> Github Repo</a>. Thanks!
+---
+📧 **Contact**
+If you have any questions or feedbacks, feel free to open a discussion or contact <b>wuyanze123@gmail.com</b>.
+"""  # noqa E501
+
+_DEV_DES = '''
+* Please refer to our repo for instructions on running gradio demo [locally](https://github.com/ToTheBeginning/PuLID/blob/main/docs/pulid_for_flux.md#local-gradio-demo)
+'''
+
+
+def create_demo(args, model_name: str, device: str = "cuda" if torch.cuda.is_available() else "cpu",
+                offload: bool = False):
+    with gr.Blocks() as demo:
+        with gr.Accordion("For Developers", open=False):
+            gr.Markdown(_DEV_DES)
+            
+        gr.Markdown(_HEADER_)
+
+        with gr.Row():
+            with gr.Column():
+                prompt = gr.Textbox(label="Prompt", value="portrait, color, cinematic")
+                id_image = gr.Image(label="ID Image")
+                id_weight = gr.Slider(0.0, 3.0, 1, step=0.05, label="id weight")
+
+                width = gr.Slider(256, 1536, 896, step=16, label="Width")
+                height = gr.Slider(256, 1536, 1152, step=16, label="Height")
+                num_steps = gr.Slider(1, 20, 20, step=1, label="Number of steps")
+                start_step = gr.Slider(0, 10, 0, step=1, label="timestep to start inserting ID")
+                guidance = gr.Slider(1.0, 10.0, 4, step=0.1, label="Guidance")
+                seed = gr.Textbox(-1, label="Seed (-1 for random)")
+                max_sequence_length = gr.Slider(128, 512, 128, step=128,
+                                                label="max_sequence_length for prompt (T5), small will be faster")
+
+                with gr.Accordion("Advanced Options (True CFG, true_cfg_scale=1 means use fake CFG, >1 means use true CFG, if using true CFG, we recommend set the guidance scale to 1)", open=False):    # noqa E501
+                    neg_prompt = gr.Textbox(
+                        label="Negative Prompt",
+                        value="bad quality, worst quality, text, signature, watermark, extra limbs")
+                    true_cfg = gr.Slider(1.0, 10.0, 1, step=0.1, label="true CFG scale")
+                    timestep_to_start_cfg = gr.Slider(0, 20, 1, step=1, label="timestep to start cfg", visible=args.dev)
+
+                generate_btn = gr.Button("Generate")
+
+            with gr.Column():
+                output_image = gr.Image(label="Generated Image", format='png')
+                seed_output = gr.Textbox(label="Used Seed")
+                intermediate_output = gr.Gallery(label='Output', elem_id="gallery", visible=args.dev)
+                gr.Markdown(_CITE_)
+
+        with gr.Row(), gr.Column():
+                gr.Markdown("## Examples")
+                example_inps = [
+                    [
+                        'a woman holding sign with glowing green text \"PuLID for FLUX\"',
+                        'example_inputs/liuyifei.png',
+                        4, 4, 2680261499100305976, 1
+                    ],
+                    [
+                        'portrait, side view',
+                        'example_inputs/liuyifei.png',
+                        4, 4, 180825677246321775, 1
+                    ],
+                    [
+                        'white-haired woman with vr technology atmosphere, revolutionary exceptional magnum with remarkable details',  # noqa E501
+                        'example_inputs/liuyifei.png',
+                        4, 4, 16942328329935464989, 1
+                    ],
+                    [
+                        'a young child is eating Icecream',
+                        'example_inputs/liuyifei.png',
+                        4, 4, 4527590969012358757, 1
+                    ],
+                    [
+                        'a man is holding a sign with text \"PuLID for FLUX\", winter, snowing, top of the mountain',
+                        'example_inputs/pengwei.jpg',
+                        4, 4, 6273700647573240909, 1
+                    ],
+                    [
+                        'portrait, candle light',
+                        'example_inputs/pengwei.jpg',
+                        4, 4, 17522759474323955700, 1
+                    ],
+                    [
+                        'profile shot dark photo of a 25-year-old male with smoke escaping from his mouth, the backlit smoke gives the image an ephemeral quality, natural face, natural eyebrows, natural skin texture, award winning photo, highly detailed face, atmospheric lighting, film grain, monochrome',  # noqa E501
+                        'example_inputs/pengwei.jpg',
+                        4, 4, 17733156847328193625, 1
+                    ],
+                    [
+                        'American Comics, 1boy',
+                        'example_inputs/pengwei.jpg',
+                        1, 4, 13223174453874179686, 1
+                    ],
+                    [
+                        'portrait, pixar',
+                        'example_inputs/pengwei.jpg',
+                        1, 4, 9445036702517583939, 1
+                    ],
+                ]
+                gr.Examples(examples=example_inps, inputs=[prompt, id_image, start_step, guidance, seed, true_cfg],
+                            label='fake CFG', cache_examples='lazy', outputs=[output_image, seed_output],
+                            fn=generate_image)
+
+                example_inps = [
+                    [
+                        'portrait, made of ice sculpture',
+                        'example_inputs/lecun.jpg',
+                        1, 1, 7717391560531186077, 5
+                    ],
+                ]
+                gr.Examples(examples=example_inps, inputs=[prompt, id_image, start_step, guidance, seed, true_cfg],
+                            label='true CFG', cache_examples='lazy', outputs=[output_image, seed_output],
+                            fn=generate_image)
+
+        generate_btn.click(
+            fn=generate_image,
+            inputs=[prompt, id_image, start_step, guidance, seed, true_cfg, width, height, num_steps, id_weight,
+                    neg_prompt, timestep_to_start_cfg, max_sequence_length],
+            outputs=[output_image, seed_output, intermediate_output],
+        )
+
+    return demo
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="PuLID for FLUX.1-dev")
+    parser.add_argument("--name", type=str, default="flux-dev", choices=list('flux-dev'),
+                        help="currently only support flux-dev")
+    parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu",
+                        help="Device to use")
+    parser.add_argument("--offload", action="store_true", help="Offload model to CPU when not in use")
+    parser.add_argument("--port", type=int, default=8080, help="Port to use")
+    parser.add_argument("--dev", action='store_true', help="Development mode")
+    parser.add_argument("--pretrained_model", type=str, help='for development')
+    args = parser.parse_args()
+
+    import huggingface_hub
+    huggingface_hub.login(os.getenv('HF_TOKEN'))
+
+    demo = create_demo(args, args.name, args.device, args.offload)
+    demo.launch()

gitattributes (8)

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+example_inputs/pengwei.jpg filter=lfs diff=lfs merge=lfs -text

pulid_pipeline_flux.py

@@ -0,0 +1,188 @@
+import gc
+
+import cv2
+import insightface
+import torch
+import torch.nn as nn
+from basicsr.utils import img2tensor, tensor2img
+from facexlib.parsing import init_parsing_model
+from facexlib.utils.face_restoration_helper import FaceRestoreHelper
+from huggingface_hub import hf_hub_download, snapshot_download
+from insightface.app import FaceAnalysis
+from safetensors.torch import load_file
+from torchvision.transforms import InterpolationMode
+from torchvision.transforms.functional import normalize, resize
+
+from eva_clip import create_model_and_transforms
+from eva_clip.constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
+from pulid.encoders_flux import IDFormer, PerceiverAttentionCA
+
+
+class PuLIDPipeline(nn.Module):
+    def __init__(self, dit, device, weight_dtype=torch.bfloat16, *args, **kwargs):
+        super().__init__()
+        self.device = device
+        self.weight_dtype = weight_dtype
+        double_interval = 2
+        single_interval = 4
+
+        # init encoder
+        self.pulid_encoder = IDFormer().to(self.device, self.weight_dtype)
+
+        num_ca = 19 // double_interval + 38 // single_interval
+        if 19 % double_interval != 0:
+            num_ca += 1
+        if 38 % single_interval != 0:
+            num_ca += 1
+        self.pulid_ca = nn.ModuleList([
+            PerceiverAttentionCA().to(self.device, self.weight_dtype) for _ in range(num_ca)
+        ])
+
+        dit.pulid_ca = self.pulid_ca
+        dit.pulid_double_interval = double_interval
+        dit.pulid_single_interval = single_interval
+
+        # preprocessors
+        # face align and parsing
+        self.face_helper = FaceRestoreHelper(
+            upscale_factor=1,
+            face_size=512,
+            crop_ratio=(1, 1),
+            det_model='retinaface_resnet50',
+            save_ext='png',
+            device=self.device,
+        )
+        self.face_helper.face_parse = None
+        self.face_helper.face_parse = init_parsing_model(model_name='bisenet', device=self.device)
+        # clip-vit backbone
+        model, _, _ = create_model_and_transforms('EVA02-CLIP-L-14-336', 'eva_clip', force_custom_clip=True)
+        model = model.visual
+        self.clip_vision_model = model.to(self.device, dtype=self.weight_dtype)
+        eva_transform_mean = getattr(self.clip_vision_model, 'image_mean', OPENAI_DATASET_MEAN)
+        eva_transform_std = getattr(self.clip_vision_model, 'image_std', OPENAI_DATASET_STD)
+        if not isinstance(eva_transform_mean, (list, tuple)):
+            eva_transform_mean = (eva_transform_mean,) * 3
+        if not isinstance(eva_transform_std, (list, tuple)):
+            eva_transform_std = (eva_transform_std,) * 3
+        self.eva_transform_mean = eva_transform_mean
+        self.eva_transform_std = eva_transform_std
+        # antelopev2
+        snapshot_download('DIAMONIK7777/antelopev2', local_dir='models/antelopev2')
+        self.app = FaceAnalysis(
+            name='antelopev2', root='.', providers=['CPUExecutionProvider']
+        )
+        self.app.prepare(ctx_id=0, det_size=(640, 640))
+        self.handler_ante = insightface.model_zoo.get_model('models/antelopev2/glintr100.onnx', providers=['CPUExecutionProvider'])
+        self.handler_ante.prepare(ctx_id=0)
+
+        gc.collect()
+        torch.cuda.empty_cache()
+
+        # self.load_pretrain()
+
+        # other configs
+        self.debug_img_list = []
+
+    def load_pretrain(self, pretrain_path=None):
+        hf_hub_download('guozinan/PuLID', 'pulid_flux_v0.9.0.safetensors', local_dir='models')
+        ckpt_path = 'models/pulid_flux_v0.9.0.safetensors'
+        if pretrain_path is not None:
+            ckpt_path = pretrain_path
+        state_dict = load_file(ckpt_path)
+        state_dict_dict = {}
+        for k, v in state_dict.items():
+            module = k.split('.')[0]
+            state_dict_dict.setdefault(module, {})
+            new_k = k[len(module) + 1:]
+            state_dict_dict[module][new_k] = v
+
+        for module in state_dict_dict:
+            print(f'loading from {module}')
+            getattr(self, module).load_state_dict(state_dict_dict[module], strict=True)
+
+        del state_dict
+        del state_dict_dict
+
+    def to_gray(self, img):
+        x = 0.299 * img[:, 0:1] + 0.587 * img[:, 1:2] + 0.114 * img[:, 2:3]
+        x = x.repeat(1, 3, 1, 1)
+        return x
+
+    def get_id_embedding(self, image, cal_uncond=False):
+        """
+        Args:
+            image: numpy rgb image, range [0, 255]
+        """
+        self.face_helper.clean_all()
+        self.debug_img_list = []
+        image_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+        # get antelopev2 embedding
+        # for k in self.app.models.keys():
+        #     self.app.models[k].session.set_providers(['CUDAExecutionProvider'])
+        face_info = self.app.get(image_bgr)
+        if len(face_info) > 0:
+            face_info = sorted(face_info, key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[
+                -1
+            ]  # only use the maximum face
+            id_ante_embedding = face_info['embedding']
+            self.debug_img_list.append(
+                image[
+                    int(face_info['bbox'][1]) : int(face_info['bbox'][3]),
+                    int(face_info['bbox'][0]) : int(face_info['bbox'][2]),
+                ]
+            )
+        else:
+            id_ante_embedding = None
+
+        # using facexlib to detect and align face
+        self.face_helper.read_image(image_bgr)
+        self.face_helper.get_face_landmarks_5(only_center_face=True)
+        self.face_helper.align_warp_face()
+        if len(self.face_helper.cropped_faces) == 0:
+            raise RuntimeError('facexlib align face fail')
+        align_face = self.face_helper.cropped_faces[0]
+        # incase insightface didn't detect face
+        if id_ante_embedding is None:
+            print('fail to detect face using insightface, extract embedding on align face')
+            # self.handler_ante.session.set_providers(['CUDAExecutionProvider'])
+            id_ante_embedding = self.handler_ante.get_feat(align_face)
+
+        id_ante_embedding = torch.from_numpy(id_ante_embedding).to(self.device, self.weight_dtype)
+        if id_ante_embedding.ndim == 1:
+            id_ante_embedding = id_ante_embedding.unsqueeze(0)
+
+        # parsing
+        input = img2tensor(align_face, bgr2rgb=True).unsqueeze(0) / 255.0
+        input = input.to(self.device)
+        parsing_out = self.face_helper.face_parse(normalize(input, [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]))[0]
+        parsing_out = parsing_out.argmax(dim=1, keepdim=True)
+        bg_label = [0, 16, 18, 7, 8, 9, 14, 15]
+        bg = sum(parsing_out == i for i in bg_label).bool()
+        white_image = torch.ones_like(input)
+        # only keep the face features
+        face_features_image = torch.where(bg, white_image, self.to_gray(input))
+        self.debug_img_list.append(tensor2img(face_features_image, rgb2bgr=False))
+
+        # transform img before sending to eva-clip-vit
+        face_features_image = resize(face_features_image, self.clip_vision_model.image_size, InterpolationMode.BICUBIC)
+        face_features_image = normalize(face_features_image, self.eva_transform_mean, self.eva_transform_std)
+        id_cond_vit, id_vit_hidden = self.clip_vision_model(
+            face_features_image.to(self.weight_dtype), return_all_features=False, return_hidden=True, shuffle=False
+        )
+        id_cond_vit_norm = torch.norm(id_cond_vit, 2, 1, True)
+        id_cond_vit = torch.div(id_cond_vit, id_cond_vit_norm)
+
+        id_cond = torch.cat([id_ante_embedding, id_cond_vit], dim=-1)
+
+        id_embedding = self.pulid_encoder(id_cond, id_vit_hidden)
+
+        if not cal_uncond:
+            return id_embedding, None
+
+        id_uncond = torch.zeros_like(id_cond)
+        id_vit_hidden_uncond = []
+        for layer_idx in range(0, len(id_vit_hidden)):
+            id_vit_hidden_uncond.append(torch.zeros_like(id_vit_hidden[layer_idx]))
+        uncond_id_embedding = self.pulid_encoder(id_uncond, id_vit_hidden_uncond)
+
+        return id_embedding, uncond_id_embedding

requirements (2).txt

@@ -0,0 +1,20 @@
+diffusers==0.25.0
+torch==2.1.0
+torchvision==0.16.0
+transformers==4.43.3
+opencv-python
+einops
+ftfy
+basicsr
+facexlib
+insightface
+onnx==1.13.1
+onnxruntime-gpu
+onnxruntime==1.14.1
+accelerate
+huggingface-hub
+timm
+SentencePiece
+fire
+safetensors
+numpy==1.24.1