import gradio as gr import torch import numpy as np from transformers import OwlViTProcessor, OwlViTForObjectDetection from torchvision import transforms from PIL import Image, ImageDraw import cv2 import torch.nn.functional as F import tempfile import os from SuperGluePretrainedNetwork.models.matching import Matching from SuperGluePretrainedNetwork.models.utils import read_image import matplotlib.pyplot as plt import matplotlib.cm as cm from io import BytesIO # Set device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Load models mixin = OwlViTForObjectDetection.from_pretrained("google/owlvit-base-patch32") processor = OwlViTProcessor.from_pretrained("google/owlvit-base-patch32") model = mixin.to(device) matching = Matching({ 'superpoint': {'nms_radius': 4, 'keypoint_threshold': 0.005, 'max_keypoints': 1024}, 'superglue': {'weights': 'outdoor', 'sinkhorn_iterations': 20, 'match_threshold': 0.2} }).eval().to(device) # Utility functions def preprocess_image(image): transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) return transform(image).unsqueeze(0) def save_array_to_temp_image(arr): rgb_arr = cv2.cvtColor(arr, cv2.COLOR_BGR2RGB) img = Image.fromarray(rgb_arr) temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.png') temp_file_name = temp_file.name temp_file.close() img.save(temp_file_name) return temp_file_name def stitch_images(images): if not images: return Image.new('RGB', (100, 100), color='gray') max_width = max([img.width for img in images]) total_height = sum(img.height for img in images) composite = Image.new('RGB', (max_width, total_height)) y_offset = 0 for img in images: composite.paste(img, (0, y_offset)) y_offset += img.height return composite def unified_matching_plot2(image0, image1, kpts0, kpts1, mkpts0, mkpts1, color, text, path=None, show_keypoints=False, fast_viz=False, opencv_display=False, opencv_title='matches', small_text=[]): # Resize images to have the same height height = min(image0.shape[0], image1.shape[0]) image0_resized = cv2.resize(image0, (int(image0.shape[1] * height / image0.shape[0]), height)) image1_resized = cv2.resize(image1, (int(image1.shape[1] * height / image1.shape[0]), height)) plt.figure(figsize=(15, 15)) plt.subplot(1, 2, 1) plt.imshow(image0_resized) plt.scatter(kpts0[:, 0], kpts0[:, 1], color='r', s=1) plt.axis('off') plt.subplot(1, 2, 2) plt.imshow(image1_resized) plt.scatter(kpts1[:, 0], kpts1[:, 1], color='r', s=1) plt.axis('off') fig, ax = plt.subplots(figsize=(20, 20)) plt.plot([mkpts0[:, 0], mkpts1[:, 0] + image0_resized.shape[1]], [mkpts0[:, 1], mkpts1[:, 1]], 'r', lw=0.5) plt.scatter(mkpts0[:, 0], mkpts0[:, 1], s=2, marker='o', color='b') plt.scatter(mkpts1[:, 0] + image0_resized.shape[1], mkpts1[:, 1], s=2, marker='o', color='g') plt.imshow(np.hstack([image0_resized, image1_resized]), aspect='auto') plt.suptitle('\n'.join(text), fontsize=20, fontweight='bold') plt.tight_layout() plt.show() buf = BytesIO() plt.savefig(buf, format='png') buf.seek(0) img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8) buf.close() img = cv2.imdecode(img_arr, 1) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) plt.close(fig) return img # Main functions def detect_and_crop(target_image, query_image, threshold=0.5, nms_threshold=0.3): target_sizes = torch.Tensor([target_image.size[::-1]]) inputs = processor(images=target_image, query_images=query_image, return_tensors="pt").to(device) with torch.no_grad(): outputs = model.image_guided_detection(**inputs) img = cv2.cvtColor(np.array(target_image), cv2.COLOR_BGR2RGB) outputs.logits = outputs.logits.cpu() outputs.target_pred_boxes = outputs.target_pred_boxes.cpu() results = processor.post_process_image_guided_detection(outputs=outputs, threshold=threshold, nms_threshold=nms_threshold, target_sizes=target_sizes) boxes, scores = results[0]["boxes"], results[0]["scores"] if len(boxes) == 0: return [], None filtered_boxes = [] for box in boxes: x1, y1, x2, y2 = [int(i) for i in box.tolist()] cropped_img = img[y1:y2, x1:x2] if cropped_img.size != 0: filtered_boxes.append(cropped_img) draw = ImageDraw.Draw(target_image) for box in boxes: draw.rectangle(box.tolist(), outline="red", width=3) return filtered_boxes, target_image def image_matching_no_pyramid(query_img, target_img, visualize=True): temp_query = save_array_to_temp_image(np.array(query_img)) temp_target = save_array_to_temp_image(np.array(target_img)) image1, inp1, scales1 = read_image(temp_target, device, [640*2], 0, True) image0, inp0, scales0 = read_image(temp_query, device, [640*2], 0, True) if image0 is None or image1 is None: return None pred = matching({'image0': inp0, 'image1': inp1}) pred = {k: v[0] for k, v in pred.items()} kpts0, kpts1 = pred['keypoints0'], pred['keypoints1'] matches, conf = pred['matches0'], pred['matching_scores0'] valid = matches > -1 mkpts0 = kpts0[valid] mkpts1 = kpts1[matches[valid]] mconf = conf[valid] color = cm.jet(mconf.detach().cpu().numpy())[:len(mkpts0)] valid_count = np.sum(valid.tolist()) mkpts0_np = mkpts0.cpu().numpy() mkpts1_np = mkpts1.cpu().numpy() try: H, inliers = cv2.findHomography(mkpts0_np, mkpts1_np, cv2.RANSAC, 5.0) except: inliers = 0 num_inliers = np.sum(inliers) if visualize: visualized_img = unified_matching_plot2( image0, image1, kpts0, kpts1, mkpts0, mkpts1, color, ['Matches'], True, False, True, 'Matches', []) else: visualized_img = None return { 'valid': [valid_count], 'inliers': [num_inliers], 'visualized_image': [visualized_img] } def check_object_in_image(query_image, target_image, threshold=50, scale_factor=[0.33, 0.66, 1]): images_to_return = [] cropped_images, bbox_image = detect_and_crop(target_image, query_image) temp_files = [save_array_to_temp_image(i) for i in cropped_images] crop_results = [image_matching_no_pyramid(query_image, Image.open(i), visualize=True) for i in temp_files] cropped_visuals = [] cropped_inliers = [] for result in crop_results: if result: for img in result['visualized_image']: cropped_visuals.append(Image.fromarray(img)) for inliers_ in result['inliers']: cropped_inliers.append(inliers_) images_to_return.append(stitch_images(cropped_visuals)) is_present = any(value >= threshold for value in cropped_inliers) return { 'is_present': is_present, 'images': images_to_return, 'object detection inliers': [int(i) for i in cropped_inliers], 'bbox_image': bbox_image, } def interface(poster_source, media_source, threshold, scale_factor): result1 = check_object_in_image(poster_source, media_source, threshold, scale_factor) if result1['is_present']: return result1 result2 = check_object_in_image(poster_source, media_source, threshold, scale_factor) return result2 if result2['is_present'] else result1 iface = gr.Interface( fn=interface, inputs=[ gr.Image(type="pil", label="Upload a Query Image (Poster)"), gr.Image(type="pil", label="Upload a Target Image (Media)"), gr.Slider(minimum=0, maximum=100, step=1, value=50, label="Threshold"), gr.CheckboxGroup(choices=[0.33, 0.66, 1.0], value=[0.33, 0.66, 1.0], label="Scale Factors") ], outputs=[ gr.JSON(label="Result") ], title="Object Detection in Image", description=""" **Instructions:** 1. **Upload a Query Image (Poster)**: Select an image file that contains the object you want to detect. 2. **Upload a Target Image (Media)**: Select an image file where you want to detect the object. 3. **Set Threshold**: Adjust the slider to set the threshold for object detection. 4. **Set Scale Factors**: Select the scale factors for image pyramid. 5. **View Results**: The result will show whether the object is present in the image along with additional details. """ ) if __name__ == "__main__": iface.launch()