File size: 5,674 Bytes
2645a2e
5d63fc3
2645a2e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88004fc
 
2645a2e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5cee4f3
 
 
 
 
2645a2e
af0c10d
5cee4f3
 
2645a2e
af0c10d
2645a2e
af0c10d
5cee4f3
 
 
 
6428557
 
2645a2e
5cee4f3
2645a2e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5d63fc3
2645a2e
 
 
f576e7d
 
 
7da477e
9d70bcc
2645a2e
 
 
 
 
 
 
 
 
 
 
5cee4f3
df91e4c
f576e7d
2645a2e
 
 
f9f60e5
 
 
2645a2e
 
88004fc
5d63fc3
 
180c6d2
6b323b9
180c6d2
5d63fc3
 
2645a2e
 
5d63fc3
2645a2e
 
 
 
 
 
 
f576e7d
2645a2e
9d70bcc
 
1a3a077
 
2645a2e
 
 
 
6428557
 
7da477e
df91e4c
2645a2e
 
df91e4c
 
2645a2e
 
9d70bcc
 
2645a2e
 
 
 
63c64d0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
import subprocess
from PIL import Image

def download_file(url, output_filename):
    command = ['wget', '-O', output_filename, '-q', url]
    subprocess.run(command, check=True)

url1 = 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_multiclass_256x256/float32/latest/selfie_multiclass_256x256.tflite'
url2 = 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite'

filename1 = 'selfie_multiclass_256x256.tflite'
filename2 = 'selfie_segmenter.tflite'

download_file(url1, filename1)
download_file(url2, filename2)

import cv2
import mediapipe as mp
import numpy as np
from mediapipe.tasks import python
from mediapipe.tasks.python import vision
import random
import gradio as gr
import spaces
import torch
from diffusers import FluxInpaintPipeline
from diffusers import FlowMatchEulerDiscreteScheduler, AutoencoderKL
from diffusers.models.transformers.transformer_flux import FluxTransformer2DModel
from transformers import CLIPTextModel, CLIPTokenizer,T5EncoderModel, T5TokenizerFast

DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

bfl_repo="black-forest-labs/FLUX.1-dev"

BG_COLOR = (255, 255, 255) # white
MASK_COLOR = (0, 0 , 0) # black

def maskHead(input):
  base_options = python.BaseOptions(model_asset_path='selfie_multiclass_256x256.tflite')
  options = vision.ImageSegmenterOptions(base_options=base_options,
                                       output_category_mask=True)

  with vision.ImageSegmenter.create_from_options(options) as segmenter:
    image = mp.Image.create_from_file(input)

    segmentation_result = segmenter.segment(image)

    hairmask = segmentation_result.confidence_masks[1]
    facemask = segmentation_result.confidence_masks[3]

    image_data = image.numpy_view()
    fg_image = np.zeros(image_data.shape, dtype=np.uint8)
    fg_image[:] = MASK_COLOR
    bg_image = np.zeros(image_data.shape, dtype=np.uint8)
    bg_image[:] = BG_COLOR

    combined_mask = np.maximum(hairmask.numpy_view(), facemask.numpy_view())

    condition = np.stack((combined_mask,) * 3, axis=-1) > 0.2
    output_image = np.where(condition, fg_image, bg_image)

    return output_image

def random_positioning(input, output_size=(1024, 1024)):
    background = cv2.imread("default.jpeg")
    if background is None:
        raise ValueError("Unable to load background image")

    background = cv2.resize(background, output_size, interpolation=cv2.INTER_AREA)
    
    if input is None:
        raise ValueError("Unable to load input image")

    scale_factor = random.uniform(0.5, 1.0)
    new_size = (int(input.shape[1] * scale_factor), int(input.shape[0] * scale_factor))
    
    resized_image = cv2.resize(input, new_size, interpolation=cv2.INTER_AREA)

    if background.shape[2] != resized_image.shape[2]:
        raise ValueError("Input image and background image must have the same number of channels")

    x_offset = random.randint(0, output_size[0] - new_size[0])
    y_offset = random.randint(0, output_size[1] - new_size[1])
    background[y_offset:y_offset+new_size[1], x_offset:x_offset+new_size[0]] = resized_image
    
    return background


def remove_background(image_path, mask):
    image = cv2.imread(image_path)
    inverted_mask = cv2.bitwise_not(mask)
    
    _, binary_mask = cv2.threshold(inverted_mask, 127, 255, cv2.THRESH_BINARY)
    
    result = np.zeros_like(image, dtype=np.uint8)
    
    result[binary_mask == 255] = image[binary_mask == 255]
    
    return result
    
pipe = FluxInpaintPipeline.from_pretrained(bfl_repo, torch_dtype=torch.bfloat16).to(DEVICE)
MAX_SEED = np.iinfo(np.int32).max
TRIGGER = "a photo of TOK"

print(dir(pipe))


@spaces.GPU(duration=100)
def execute(image, prompt, debug=False):
  if not prompt :
        gr.Info("Please enter a text prompt.")
        return None

  if not image :
        gr.Info("Please upload a image.")
        return None

  img = cv2.imread(image)
  img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

  imgs = [ random_positioning(img), random_positioning(img)]

  pipe.load_lora_weights("XLabs-AI/flux-RealismLora", weight_name='lora.safetensors')
  response = []

  for image in range(len(imgs)):
    seed_slicer = random.randint(0, MAX_SEED)
    generator = torch.Generator().manual_seed(seed_slicer)

    current_img = imgs[image]
    cv2.imwrite('base_image.jpg', current_img)
    cv2.imwrite("mask.jpg", maskHead('base_image.jpg'))
    
    im = Image.open('base_image.jpg')
    np_arr = np.array(im)
    rgb_image = cv2.cvtColor(np_arr, cv2.COLOR_BGR2RGB)
    im =  Image.fromarray(rgb_image)
    mask = Image.open('mask.jpg')

    result = pipe(
            prompt=f"{prompt} {TRIGGER}",
            image=im,
            mask_image=mask,
            width=1024,
            height=1024,
            strength=0.85,
            generator=generator,
            num_inference_steps=28,
            max_sequence_length=256,
            joint_attention_kwargs={"scale": 0.9},
        ).images[0]

    if debug:
        response.append(im)
        response.append(mask)
    response.append(result)

  return response

# Created by Fountai
# https://x.com/EuFountai
description = "This is an unofficial implementation of the ip face adapter for FLUX DEV and does not explicitly follow the ip face model, I created a wrapper with inpaint and mediapipe, I like to call Fake IP Adapter"
title = "Flux IP Face Adapter"
iface = gr.Interface(
    fn=execute,
    description=description,
    title=title,
    inputs=[
        gr.Image(type="filepath"),
        gr.Textbox(label="Prompt"),
        gr.Checkbox(label="Debug Mode")
    ],
    outputs="gallery"
)

iface.launch()