import torch import random import numpy as np from PIL import Image from collections import defaultdict import os import spaces # Mentioning detectron2 as a dependency directly in requirements.txt tries to install detectron2 before torch and results in an error even if torch is listed as a dependency before detectron2. # Hence, installing detectron2 this way when using Gradio HF spaces. os.system('pip install git+https://github.com/facebookresearch/detectron2.git') from detectron2.data import MetadataCatalog from detectron2.utils.visualizer import ColorMode, Visualizer from color_palette import ade_palette from transformers import Mask2FormerImageProcessor, Mask2FormerForUniversalSegmentation @spaces.GPU def load_model_and_processor(model_ckpt: str): device = "cuda" if torch.cuda.is_available() else "cpu" model = Mask2FormerForUniversalSegmentation.from_pretrained(model_ckpt).to(torch.device(device)) model.eval() image_preprocessor = Mask2FormerImageProcessor.from_pretrained(model_ckpt) return model, image_preprocessor @spaces.GPU def load_default_ckpt(): default_ckpt = "facebook/mask2former-swin-tiny-coco-panoptic" return default_ckpt @spaces.GPU def draw_panoptic_segmentation(predicted_segmentation_map, seg_info, image): metadata = MetadataCatalog.get("coco_2017_val_panoptic") for res in seg_info: res['category_id'] = res.pop('label_id') pred_class = res['category_id'] isthing = pred_class in metadata.thing_dataset_id_to_contiguous_id.values() res['isthing'] = bool(isthing) visualizer = Visualizer(np.array(image)[:, :, ::-1], metadata=metadata, instance_mode=ColorMode.IMAGE) out = visualizer.draw_panoptic_seg_predictions( predicted_segmentation_map.cpu(), seg_info, alpha=0.5 ) output_img = Image.fromarray(out.get_image()) labels = [res['category_id'] for res in seg_info] return output_img, labels @spaces.GPU def predict_masks(input_img_path: str): #load model and image processor default_ckpt = load_default_ckpt() model, image_processor = load_model_and_processor(default_ckpt) ## pass input image through image processor image = Image.open(input_img_path) inputs = image_processor(images=image, return_tensors="pt") # Move inputs to the same device as the model inputs = {name: tensor.to(model.device) for name, tensor in inputs.items()} ## pass inputs to model for prediction with torch.no_grad(): outputs = model(**inputs) result = image_processor.post_process_panoptic_segmentation(outputs, target_sizes=[image.size[::-1]])[0] predicted_segmentation_map = result["segmentation"] seg_info = result['segments_info'] output_result, labels = draw_panoptic_segmentation(predicted_segmentation_map, seg_info, image) output_heading = "Panoptic Segmentation Output" return output_result, output_heading, labels @spaces.GPU def get_mask_for_label(results, label): import numpy as np from PIL import Image mask = (results['segmentation'].numpy() == label) visual_mask = (mask * 255).astype(np.uint8) visual_mask = Image.fromarray(visual_mask) return visual_mask