File size: 18,029 Bytes
b1532b0 2abf116 ff6bf82 2abf116 a4dc223 f30185d 2abf116 f30185d 2abf116 ff6bf82 2abf116 b1532b0 f932259 b1532b0 39ca3cd ff6bf82 b1532b0 f932259 6e0ffb7 2abf116 210730f b1532b0 67fb9c5 b1532b0 2abf116 210730f 2abf116 210730f 2abf116 210730f 2abf116 210730f e09fed0 ff6bf82 39ca3cd 3bda041 4724a1e 3bda041 1f79208 6845b01 b1532b0 ff6bf82 3bda041 2abf116 f30185d a792718 2abf116 64af198 2abf116 64af198 2abf116 210730f b3333a0 210730f 7557387 210730f 540630a 210730f 540630a 0fdfbbc 540630a 210730f 2abf116 ca2d13e b1532b0 ca2d13e e09fed0 ca2d13e e09fed0 b1532b0 a4dc223 ff6bf82 fffe6fa ff6bf82 fffe6fa ff6bf82 fffe6fa ff6bf82 39ca3cd ff6bf82 5012113 a4dc223 2abf116 a4dc223 e3ccbe7 0fdfbbc a4dc223 2abf116 a4dc223 f30185d a4dc223 5012113 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 |
import ast
import json
import base64
import spaces
import requests
import numpy as np
import gradio as gr
from PIL import Image
from io import BytesIO
import face_recognition
from turtle import title
from openai import OpenAI
from collections import Counter
from transformers import pipeline
import urllib.request
from transformers import YolosImageProcessor, YolosForObjectDetection
import torch
import matplotlib.pyplot as plt
from torchvision.transforms import ToTensor, ToPILImage
client = OpenAI()
pipe = pipeline("zero-shot-image-classification", model="patrickjohncyh/fashion-clip")
color_file_path = 'color_config.json'
attributes_file_path = 'attributes_config.json'
import os
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
# Open and read the COLOR JSON file
with open(color_file_path, 'r') as file:
color_data = json.load(file)
# Open and read the ATTRIBUTES JSON file
with open(attributes_file_path, 'r') as file:
attributes_data = json.load(file)
COLOURS_DICT = color_data['color_mapping']
ATTRIBUTES_DICT = attributes_data['attribute_mapping']
def shot(input, category):
subColour,mainColour,score = get_colour(ast.literal_eval(str(input)),category)
common_result = get_predicted_attributes(ast.literal_eval(str(input)),category)
openai_parsed_response = get_openAI_tags(ast.literal_eval(str(input)))
face_embeddings = get_face_embeddings(ast.literal_eval(str(input)))
cropped_images = get_cropped_images(ast.literal_eval(str(input)),category)
print(cropped_images)
return {
"colors":{
"main":mainColour,
"sub":subColour,
"score":round(score*100,2)
},
"attributes":common_result,
"image_mapping":openai_parsed_response,
"face_embeddings":face_embeddings
}
@spaces.GPU
def get_colour(image_urls, category):
colourLabels = list(COLOURS_DICT.keys())
for i in range(len(colourLabels)):
colourLabels[i] = colourLabels[i] + " clothing: " + category
responses = pipe(image_urls, candidate_labels=colourLabels)
# Get the most common colour
mainColour = responses[0][0]['label'].split(" clothing:")[0]
if mainColour not in COLOURS_DICT:
return None, None, None
# Add category to the end of each label
labels = COLOURS_DICT[mainColour]
for i in range(len(labels)):
labels[i] = labels[i] + " clothing: " + category
# Run pipeline in one go
responses = pipe(image_urls, candidate_labels=labels)
subColour = responses[0][0]['label'].split(" clothing:")[0]
return subColour, mainColour, responses[0][0]['score']
@spaces.GPU
def get_predicted_attributes(image_urls, category):
# Get the predicted attributes for the image
# attributes = get_category_attributes(category)
attributes = list(ATTRIBUTES_DICT.get(category,{}).keys())
# Mapping of possible values per attribute
common_result = []
for attribute in attributes:
# values = get_attribute_values(attribute, category)
values = ATTRIBUTES_DICT.get(category,{}).get(attribute,[])
if len(values) == 0:
continue
# Adjust labels for the pipeline to be in format: "{attr}: {value}, clothing: {category}"
attribute = attribute.replace("colartype", "collar").replace("sleevelength", "sleeve length").replace("fabricstyle", "fabric")
values = [f"{attribute}: {value}, clothing: {category}" for value in values]
# Get the predicted values for the attribute
responses = pipe(image_urls, candidate_labels=values)
result = [response[0]['label'].split(", clothing:")[0] for response in responses]
# If attribute is details, then get the top 2 most common labels
if attribute == "details":
result += [response[1]['label'].split(", clothing:")[0] for response in responses]
common_result.append(Counter(result).most_common(2))
else:
common_result.append(Counter(result).most_common(1))
# Clean up the results into one long string
for i, result in enumerate(common_result):
common_result[i] = ", ".join([f"{x[0]}" for x in result])
result = {}
# Iterate through the list and split each item into key and value
for item in common_result:
# Split by ': ' to separate the key and value
key, value = item.split(': ', 1)
# Add to the dictionary
result[key] = value
return result
def get_openAI_tags(image_urls):
# Create list containing JSONs of each image URL
imageList = []
for image in image_urls:
imageList.append({"type": "image_url", "image_url": {"url": image}})
openai_response = client.chat.completions.create(
model="gpt-4o",
messages=[
{
"role": "system",
"content": [
{
"type": "text",
"text": "You're a tagging assistant, you will help label and tag product pictures for my online e-commerce platform. Your tasks will be to return which angle the product images were taken from. You will have to choose from 'full-body', 'half-body', 'side', 'back', or 'zoomed' angles. You should label each of the images with one of these labels depending on which you think fits best (ideally, every label should be used at least once, but only if there are 5 or more images), and should respond with an unformatted dictionary where the key is a string representation of the url index of the url and the value is the assigned label."
}
]
},
{
"role": "user",
"content": imageList
},
],
temperature=1,
max_tokens=500,
top_p=1,
frequency_penalty=0,
presence_penalty=0
)
response = json.loads(openai_response.choices[0].message.content)
return response
@spaces.GPU
def get_face_embeddings(image_urls):
# Initialize a dictionary to store the face encodings or errors
results = {}
# Loop through each image URL
for index, url in enumerate(image_urls):
try:
# Try to download the image from the URL
response = requests.get(url)
# Raise an exception if the response is not successful
response.raise_for_status()
# Load the image using face_recognition
image = face_recognition.load_image_file(BytesIO(response.content))
# Get the face encodings for all faces in the image
face_encodings = face_recognition.face_encodings(image)
# If no faces are detected, store an empty list
if not face_encodings:
results[str(index)] = []
else:
# Otherwise, store the first face encoding as a list
results[str(index)] = face_encodings[0].tolist()
except Exception as e:
# If any error occurs during the download or processing, store the error message
results[str(index)] = f"Error processing image: {str(e)}"
return results
# new
ACCURACY_THRESHOLD = 0.86
def open_image_from_url(url):
# Fetch the image from the URL
response = requests.get(url, stream=True)
response.raise_for_status() # Check if the request was successful
# Open the image using PIL
image = Image.open(BytesIO(response.content))
return image
# Add the main data to the session state
main = [['Product Id', 'Sku', 'Color', 'Images', 'Status', 'Category', 'Text']]
# This is the order of the categories list. NO NOT CHANGE. Just for visualization purposes
cats = ['shirt, blouse', 'top, t-shirt, sweatshirt', 'sweater', 'cardigan', 'jacket', 'vest', 'pants', 'shorts', 'skirt', 'coat', 'dress', 'jumpsuit', 'cape', 'glasses', 'hat', 'headband, head covering, hair accessory', 'tie', 'glove', 'watch', 'belt', 'leg warmer', 'tights, stockings', 'sock', 'shoe', 'bag, wallet', 'scarf', 'umbrella', 'hood', 'collar', 'lapel', 'epaulette', 'sleeve', 'pocket', 'neckline', 'buckle', 'zipper', 'applique', 'bead', 'bow', 'flower', 'fringe', 'ribbon', 'rivet', 'ruffle', 'sequin', 'tassel']
filter = ['dress', 'jumpsuit', 'cape', 'glasses', 'hat', 'headband, head covering, hair accessory', 'tie', 'glove', 'watch', 'belt', 'leg warmer', 'tights, stockings', 'sock', 'shoe', 'scarf', 'umbrella', 'hood', 'collar', 'lapel', 'epaulette', 'sleeve', 'pocket', 'neckline', 'buckle', 'zipper', 'applique', 'bead', 'bow', 'flower', 'fringe', 'ribbon', 'rivet', 'ruffle', 'sequin', 'tassel']
# 0 for full body, 1 for upper body, 2 for lower body, 3 for over body (jacket, coat, etc), 4 for accessories
yolo_mapping = {
'shirt, blouse': 3,
'top, t-shirt, sweatshirt' : 1,
'sweater': 1,
'cardigan': 1,
'jacket': 3,
'vest': 1,
'pants': 2,
'shorts': 2,
'skirt': 2,
'coat': 3,
'dress': 0,
'jumpsuit': 0,
'bag, wallet': 4
}
# First line full body, second line upper body, third line lower body, fourth line over body, fifth line accessories
label_mapping = [
['women-dress-mini', 'women-dress-dress', 'women-dress-maxi', 'women-dress-midi', 'women-playsuitsjumpsuits-playsuit', 'women-playsuitsjumpsuits-jumpsuit', 'women-coords-coords', 'women-swimwear-onepieces', 'women-swimwear-bikinisets'],
['women-sweatersknits-cardigan', 'women-top-waistcoat', 'women-top-blouse', 'women-sweatersknits-blouse', 'women-sweatersknits-sweater', 'women-top-top', 'women-loungewear-hoodie', 'women-top-camistanks', 'women-top-tshirt', 'women-top-croptop', 'women-loungewear-sweatshirt', 'women-top-body'],
['women-loungewear-joggers', 'women-bottom-trousers', 'women-bottom-leggings', 'women-bottom-jeans', 'women-bottom-shorts', 'women-bottom-skirt', 'women-loungewear-activewear', 'women-bottom-joggers'],
['women-top-shirt', 'women-outwear-coatjacket', 'women-outwear-blazer', 'women-outwear-coatjacket', 'women-outwear-kimonos'],
['women-accessories-bags']
]
MODEL_NAME = "valentinafeve/yolos-fashionpedia"
feature_extractor = YolosImageProcessor.from_pretrained('hustvl/yolos-small')
model = YolosForObjectDetection.from_pretrained(MODEL_NAME).to(device)
def get_category_index(category):
# Find index of label mapping
for i, labels in enumerate(label_mapping):
if category in labels:
break
return i
def get_yolo_index(category):
# Find index of yolo mapping
return yolo_mapping[category]
def fix_channels(t):
"""
Some images may have 4 channels (transparent images) or just 1 channel (black and white images), in order to let the images have only 3 channels. I am going to remove the fourth channel in transparent images and stack the single channel in back and white images.
:param t: Tensor-like image
:return: Tensor-like image with three channels
"""
if len(t.shape) == 2:
return ToPILImage()(torch.stack([t for i in (0, 0, 0)]))
if t.shape[0] == 4:
return ToPILImage()(t[:3])
if t.shape[0] == 1:
return ToPILImage()(torch.stack([t[0] for i in (0, 0, 0)]))
return ToPILImage()(t)
def idx_to_text(i):
return cats[i]
# Random colors used for visualization
COLORS = [[0.000, 0.447, 0.741], [0.850, 0.325, 0.098], [0.929, 0.694, 0.125],
[0.494, 0.184, 0.556], [0.466, 0.674, 0.188], [0.301, 0.745, 0.933]]
# for output bounding box post-processing
def box_cxcywh_to_xyxy(x):
x_c, y_c, w, h = x.unbind(1)
b = [(x_c - 0.5 * w), (y_c - 0.5 * h),
(x_c + 0.5 * w), (y_c + 0.5 * h)]
return torch.stack(b, dim=1)
def rescale_bboxes(out_bbox, size):
img_w, img_h = size
b = box_cxcywh_to_xyxy(out_bbox)
b = b * torch.tensor([img_w, img_h, img_w, img_h], dtype=torch.float32)
return b
def plot_results(pil_img, prob, boxes):
plt.figure(figsize=(16,10))
plt.imshow(pil_img)
ax = plt.gca()
colors = COLORS * 100
i = 0
crops = []
crop_classes = []
for p, (xmin, ymin, xmax, ymax), c in zip(prob, boxes.tolist(), colors):
cl = p.argmax()
# Save each box as an image
box_img = pil_img.crop((xmin, ymin, xmax, ymax))
crops.append(box_img)
crop_classes.append(idx_to_text(cl))
ax.add_patch(plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin,
fill=False, color=c, linewidth=3))
ax.text(xmin, ymin, idx_to_text(cl), fontsize=10,
bbox=dict(facecolor=c, alpha=0.8))
i += 1
# Remove white padding all around the image
plt.axis('off')
plt.subplots_adjust(left=0, right=1, top=1, bottom=0)
output_img = plt.gcf()
plt.close()
return output_img, crops, crop_classes
def visualize_predictions(image, outputs, threshold=0.8):
# Keep only predictions with confidence >= threshold
probas = outputs.logits.softmax(-1)[0, :, :-1]
keep = probas.max(-1).values > threshold
# Convert predicted boxes from [0; 1] to image scales
bboxes_scaled = rescale_bboxes(outputs.pred_boxes[0, keep].cpu(), image.size)
# Get filtered probabilities and boxes based on the filter list
filter_set = set(filter)
filtered_probas_boxes = [
(proba, box) for proba, box in zip(probas[keep], bboxes_scaled)
if idx_to_text(proba.argmax()) not in filter_set
]
# If there is a jumpsuit or dress detected, remove them if there are other clothes detected
contains_jumpsuit_or_dress = any(idx_to_text(proba.argmax()) in ["jumpsuit", "dress"] for proba, _ in filtered_probas_boxes)
if contains_jumpsuit_or_dress and len(filtered_probas_boxes) > 1:
filtered_probas_boxes = [
(proba, box) for proba, box in filtered_probas_boxes
if idx_to_text(proba.argmax()) not in ["jumpsuit", "dress"]
]
# Remove duplicates: Only keep one box per class
unique_classes = set()
unique_filtered_probas_boxes = []
for proba, box in filtered_probas_boxes:
class_text = idx_to_text(proba.argmax())
if class_text not in unique_classes:
unique_classes.add(class_text)
unique_filtered_probas_boxes.append((proba, box))
# If there are remaining filtered probabilities, plot results
output_img = None
crops = None
crop_classes = None
if unique_filtered_probas_boxes:
final_probas, final_boxes = zip(*unique_filtered_probas_boxes)
output_img, crops, crop_classes = plot_results(image, list(final_probas), torch.stack(final_boxes))
# Return the classes of the detected objects
return [proba.argmax().item() for proba, _ in unique_filtered_probas_boxes], output_img, crops, crop_classes
@spaces.GPU
def get_objects(image, threshold=0.8):
class_counts = {}
image = fix_channels(ToTensor()(image))
image = image.resize((600, 800))
inputs = feature_extractor(images=image, return_tensors="pt")
outputs = model(**inputs.to(device))
detected_classes, output_img, crops, crop_classes = visualize_predictions(image, outputs, threshold=threshold)
for cl in detected_classes:
class_name = idx_to_text(cl)
if class_name not in class_counts:
class_counts[class_name] = 0
class_counts[class_name] += 1
if crop_classes is not None:
crop_classes = [get_yolo_index(c) for c in crop_classes]
return class_counts, output_img, crops, crop_classes
def get_cropped_images(images,category):
cropped_list = []
resultsPerCategory = {}
for num, image in enumerate(images):
image = open_image_from_url(image)
class_counts, output_img, cropped_images, cropped_classes = get_objects(image, 0.37)
print(cropped_images)
if not class_counts:
continue
# Get the inverse category as any other mapping label except the current one corresponding category
inverse_category = [label for i, labels in enumerate(label_mapping) for label in labels if i != get_category_index(category) and i != 0]
# If category is a cardigan, we don't recommend category indices 1 and 3
if category == 'women-sweatersknits-cardigan':
inverse_category = [label for i, labels in enumerate(label_mapping) for label in labels if i != get_category_index(category) and i != 1 and i != 3]
for i, image in enumerate(cropped_images):
cropped_category = cropped_classes[i]
print(cropped_category, cropped_classes[i], get_category_index(category))
specific_category = label_mapping[cropped_category]
if cropped_category == get_category_index(category):
continue
cropped_list.append(image)
return cropped_list
# Define the Gradio interface with the updated components
iface = gr.Interface(
fn=shot,
inputs=[
gr.Textbox(label="Image URLs (starting with http/https) comma seperated "),
gr.Textbox(label="Category")
],
outputs="text" ,
examples=[
[['https://d2q1sfov6ca7my.cloudfront.net/eyJidWNrZXQiOiAiaGljY3VwLWltYWdlLWhvc3RpbmciLCAia2V5IjogIlc4MDAwMDAwMTM0LU9SL1c4MDAwMDAwMTM0LU9SLTEuanBnIiwgImVkaXRzIjogeyJyZXNpemUiOiB7IndpZHRoIjogODAwLCAiaGVpZ2h0IjogMTIwMC4wLCAiZml0IjogIm91dHNpZGUifX19',
'https://d2q1sfov6ca7my.cloudfront.net/eyJidWNrZXQiOiAiaGljY3VwLWltYWdlLWhvc3RpbmciLCAia2V5IjogIlc4MDAwMDAwMTM0LU9SL1c4MDAwMDAwMTM0LU9SLTIuanBnIiwgImVkaXRzIjogeyJyZXNpemUiOiB7IndpZHRoIjogODAwLCAiaGVpZ2h0IjogMTIwMC4wLCAiZml0IjogIm91dHNpZGUifX19',
'https://d2q1sfov6ca7my.cloudfront.net/eyJidWNrZXQiOiAiaGljY3VwLWltYWdlLWhvc3RpbmciLCAia2V5IjogIlc4MDAwMDAwMTM0LU9SL1c4MDAwMDAwMTM0LU9SLTMuanBnIiwgImVkaXRzIjogeyJyZXNpemUiOiB7IndpZHRoIjogODAwLCAiaGVpZ2h0IjogMTIwMC4wLCAiZml0IjogIm91dHNpZGUifX19'], "women-top-shirt"]],
description="Add an image URL (starting with http/https) or upload a picture, and provide a list of labels separated by commas.",
title="Full product flow"
)
# Launch the interface
iface.launch()
|