Upload 114 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+regionspot/data/__pycache__/v3det_categories.cpython-38.pyc filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,419 @@
+import gradio as gr
+import torch
+from PIL import ImageDraw
+from PIL import Image
+import numpy as np
+from torchvision.transforms import ToPILImage
+
+import matplotlib.pyplot as plt
+import cv2
+from regionspot.modeling.regionspot import build_regionspot_model
+from regionspot import RegionSpot_Predictor
+from regionspot import SamAutomaticMaskGenerator
+import ast
+
+fdic = {
+#     "family": "Impact",
+#     "style": "italic",
+    "size": 15,
+#     "color": "yellow",
+#     "weight": "bold",
+}
+
+def show_mask(mask, ax, random_color=False):
+    if random_color:
+        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)
+    else:
+        color = np.array([30/255, 144/255, 255/255, 0.6])
+    h, w = mask.shape[-2:]
+    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
+    ax.imshow(mask_image)
+
+# Function to show points on an image
+def show_points(coords, labels, ax, marker_size=375):
+    pos_points = coords[labels == 1]
+    neg_points = coords[labels == 0]
+    ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)
+    ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)
+
+# Function to show bounding boxes on an image
+def show_box(box, ax):
+    x0, y0 = box[0], box[1]
+    w, h = box[2] - x0, box[3] - y0
+    ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor='none', lw=2))
+
+def auto_show_box(box, label, ax):
+    x0, y0 = box[0], box[1]
+    w, h =box[2], box[3]
+    ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0,0,0,0), lw=2))   
+    ax.text(x0,y0,f"{label}", fontdict=fdic)
+    
+def show_anns(image, anns, custom_vocabulary):
+    if anns == False:
+        plt.imshow(image)
+        ax = plt.gca()
+        ax.set_autoscale_on(False)
+        ax.imshow(image) 
+        pic = plt.gcf()
+        pic.canvas.draw()
+        w,h = pic.canvas.get_width_height()
+        image = Image.frombytes('RGB', (w,h), pic.canvas.tostring_rgb())
+        return image
+        
+    plt.imshow(image)
+    if len(anns) == 0:
+        return
+    sorted_anns = sorted(anns, key=(lambda x: x['area']), reverse=True)
+    ax = plt.gca()
+    ax.set_autoscale_on(False)
+
+    img = np.ones((sorted_anns[0]['segmentation'].shape[0], sorted_anns[0]['segmentation'].shape[1], 4))
+    img[:,:,3] = 0
+    for ann in sorted_anns:
+        l = custom_vocabulary[int(ann['pred_class'])]
+        if l != 'background':
+            m = ann['segmentation']
+            color_mask = np.concatenate([np.random.random(3), [0.35]])
+            img[m] = color_mask
+            b = ann['bbox']
+            auto_show_box(b,l, ax)
+    ax.imshow(img) 
+    ax.axis('off')
+    pic = plt.gcf()
+    pic.canvas.draw()
+    w,h = pic.canvas.get_width_height()
+    image = Image.frombytes('RGB', (w,h), pic.canvas.tostring_rgb())
+    return image
+    
+def process_box(image, input_box, masks, mask_iou_score, class_score, class_index, custom_vocabulary):
+    # Extract class name and display image with masks and box
+    fig, ax = plt.subplots(figsize=(10, 10))
+    ax.imshow(image)
+    for idx in range(len(input_box)):
+        show_mask(masks[idx], ax)  
+        show_box(input_box[idx], ax)  # Assuming box_prompt contains all your boxes
+    # You might want to modify the text display for multiple classes as well
+        class_name = custom_vocabulary[int(class_index[idx])]
+        ax.text(input_box[idx][0], input_box[idx][1] - 10, class_name, color='white', fontsize=14, bbox=dict(facecolor='green', edgecolor='green', alpha=0.6))
+
+    ax.axis('off')
+    pic = plt.gcf()
+    pic.canvas.draw()
+    w,h = pic.canvas.get_width_height()
+    image = Image.frombytes('RGB', (w,h), pic.canvas.tostring_rgb())
+    return image
+
+device = torch.device(
+    "cuda"
+    if torch.cuda.is_available()
+    else "mps"
+    if torch.backends.mps.is_available()
+    else "cpu"
+)
+
+# Description
+title = "<center><strong><font size='8'> RegionSpot: Recognize Any Regions </font></strong></center>"
+
+description_e = """ This is a demo on Github project [Recognize Any Regions](https://github.com/Surrey-UPLab/Recognize-Any-Regions). Welcome to give a star to it.
+                
+              """
+
+description_p = """ This is a demo on Github project [Recognize Any Regions](https://github.com/Surrey-UPLab/Recognize-Any-Regions). Welcome to give a star to it.
+                
+              """
+description_b = """ This is a demo on Github project [Recognize Any Regions](https://github.com/Surrey-UPLab/Recognize-Any-Regions). Welcome to give a star to it.
+                
+              """
+
+examples = [["examples/dogs.jpg"], ["examples/fruits.jpg"], ["examples/flowers.jpg"],
+            ["examples/000000190756.jpg"], ["examples/image.jpg"], ["examples/000000263860.jpg"],
+           ["examples/000000298738.jpg"], ["examples/000000027620.jpg"], ["examples/000000112634.jpg"],
+            ["examples/000000377814.jpg"], ["examples/000000516143.jpg"]]
+
+default_example = examples[0]
+
+css = "h1 { text-align: center } .about { text-align: justify; padding-left: 10%; padding-right: 10%; }"
+    
+def segment_sementic(image, text):
+    mask_threshold = 0.0
+    img = image
+    coor = np.nonzero(img["mask"])
+    coor[0].sort()
+    xmin = coor[0][0]
+    xmax = coor[0][-1]
+    coor[1].sort()
+    ymin = coor[1][0]
+    ymax = coor[1][-1]
+    input_box = np.array([[ymin, xmin, ymax, xmax]])
+    
+    image = img["image"]
+    input_image = np.asarray(image)
+    
+    ckpt_path = 'regionspot_bl_336.pth'
+    clip_type = 'CLIP_400M_Large_336'
+    # clip_input_size = 336
+    clip_input_size = 224
+    text = text.split(',')
+    custom_vocabulary = text
+    # Build and initialize the model
+    model, msg = build_regionspot_model(is_training=False, image_size=clip_input_size, clip_type=clip_type, pretrain_ckpt=ckpt_path, 
+                                        custom_vocabulary=custom_vocabulary)
+    # Create predictor and set image
+    predictor = RegionSpot_Predictor(model.cuda())
+    predictor.set_image(input_image, clip_input_size=clip_input_size)
+
+    masks, mask_iou_score, class_score, class_index = predictor.predict(
+    point_coords=None,
+    point_labels=None,
+    box=input_box,
+    multimask_output=False,
+    mask_threshold = mask_threshold,
+)    
+    fig = process_box(input_image, input_box,masks, mask_iou_score, class_score, class_index, custom_vocabulary)
+    
+    torch.cuda.empty_cache()
+    torch.cuda.empty_cache()
+    torch.cuda.empty_cache()
+    torch.cuda.empty_cache()
+    
+    return fig
+
+def text_segment_sementic(image, text, conf_threshold, box_threshold, crop_n_layers, crop_nms_threshold):  
+    mask_threshold = 0.0
+    image = image
+    input_image = np.asarray(image)
+    text = text.split(',')
+    
+    textP = ['background']
+    text = textP + text
+    
+    custom_vocabulary = text
+    ckpt_path = 'regionspot_bl_336.pth'
+    clip_type = 'CLIP_400M_Large_336'
+    clip_input_size = 336
+#     clip_input_size = 224
+    model, msg = build_regionspot_model(is_training=False, image_size=clip_input_size, clip_type=clip_type, pretrain_ckpt=ckpt_path, 
+                                        custom_vocabulary=custom_vocabulary)
+    mask_generator = SamAutomaticMaskGenerator(model.cuda(),
+#                                                crop_thresh=iou_threshold,
+                                               box_thresh=conf_threshold,mask_threshold=mask_threshold, 
+                                               box_nms_thresh=box_threshold, crop_n_layers=crop_n_layers, crop_nms_thresh= crop_nms_threshold)
+    masks = mask_generator.generate(input_image)
+
+    fig = show_anns(input_image, masks, custom_vocabulary)
+    
+    torch.cuda.empty_cache()
+    torch.cuda.empty_cache()
+    torch.cuda.empty_cache()
+    torch.cuda.empty_cache()
+    
+    return fig
+
+def point_segment_sementic(image, text, box_threshold, crop_nms_threshold):                               
+    global global_points
+    global global_point_label
+    global image_temp
+
+    mask_threshold = 0.0
+    input_image = image_temp
+    output_image = np.asarray(image)
+    ckpt_path = 'regionspot_bl_336.pth'
+    clip_type = 'CLIP_400M_Large_336'
+    clip_input_size = 336
+#     clip_input_size = 224
+    text = text.split(',')
+    
+    textP = ['background']
+    text = textP + text
+    
+    custom_vocabulary = text
+    model, msg = build_regionspot_model(is_training=False, image_size=clip_input_size, clip_type=clip_type, pretrain_ckpt=ckpt_path,
+                                        custom_vocabulary=custom_vocabulary)
+    mask_generator = SamAutomaticMaskGenerator(model.cuda(),
+                                               crop_thresh=0.0,
+                                               box_thresh=0.0, 
+                                               mask_threshold=mask_threshold, 
+                                               box_nms_thresh=box_threshold, crop_nms_thresh= crop_nms_threshold)
+    masks = mask_generator.generate_point(input_image,point=np.asarray(global_points), label=np.asarray(global_point_label))
+    
+    fig = show_anns(output_image, masks, custom_vocabulary)
+    
+    torch.cuda.empty_cache()
+    torch.cuda.empty_cache()
+    torch.cuda.empty_cache()
+    torch.cuda.empty_cache()
+    
+    return fig
+
+def get_points_with_draw(image, label, evt: gr.SelectData):
+    global global_points
+    global global_point_label
+    global image_temp
+    
+    if global_point_label == []:
+        image_temp = np.asarray(image)
+
+    x, y = evt.index[0], evt.index[1]
+    point_radius, point_color = 15, (255, 255, 0) if label == 'Mask' else (255, 0, 255)
+    global_points.append([x, y])
+    global_point_label.append(1 if label == 'Mask' else 0)
+    
+    draw = ImageDraw.Draw(image)
+    draw.ellipse([(x - point_radius, y - point_radius), (x + point_radius, y + point_radius)], fill=point_color)
+    return image
+
+
+cond_img_p = gr.Image(label="Input with points", value="examples/dogs.jpg", type='pil')
+cond_img_t = gr.Image(label="Input with text", value="examples/dogs.jpg", type='pil')
+cond_img_b = gr.Image(label="Input with box", type="pil",tool='sketch')
+img_p = gr.Image(label="Input with points P", type='pil')
+
+segm_img_p = gr.Image(label="Segmented Image with points", interactive=False, type='pil')
+segm_img_t = gr.Image(label="Segmented Image with text", interactive=False, type='pil')
+segm_img_b = gr.Image(label="Segmented Image with box", interactive=False, type='pil')
+
+global_points = []
+global_point_label = []
+image_temp = np.array([])
+
+with gr.Blocks(css=css, title='Region Spot') as demo:
+    with gr.Row():
+        with gr.Column(scale=1):
+            # Title
+            gr.Markdown(title)
+
+    with gr.Tab("Points mode"):
+        # Images
+        with gr.Row(variant="panel"):
+            with gr.Column(scale=1):
+                cond_img_p.render()
+
+            with gr.Column(scale=1):
+                segm_img_p.render()
+                
+        # Submit & Clear
+        with gr.Row():
+            with gr.Column():
+                with gr.Row():
+                    with gr.Column():
+                        add_or_remove = gr.Radio(["Mask", "Background"], value="Mask", label="Point_label (foreground/background)")
+                        text_box_p = gr.Textbox(label="vocabulary", value="dog,cat")
+                    with gr.Column():
+                        segment_btn_p = gr.Button("Segment with points prompt", variant='primary')
+                        clear_btn_p = gr.Button("Clear", variant='secondary')
+
+                gr.Markdown("Try some of the examples below")
+                gr.Examples(examples=examples,
+                            inputs=[cond_img_t],
+                            examples_per_page=18)
+
+            with gr.Column():
+                with gr.Accordion("Advanced options", open=True):
+                    box_threshold_p = gr.Slider(0.0, 0.9, 0.7, step=0.05, label='box threshold', info='box nms threshold')
+                    crop_threshold_p = gr.Slider(0.0, 0.9, 0.7, step=0.05, label='crop threshold', info='crop nms threshold')
+                # Description
+                gr.Markdown(description_p)
+    cond_img_p.select(get_points_with_draw, [cond_img_p, add_or_remove], cond_img_p)
+    segment_btn_p.click(point_segment_sementic,
+                        inputs=[
+                            cond_img_p,
+                            text_box_p,
+                            box_threshold_p,
+                            crop_threshold_p,
+                        ],
+                        outputs=[segm_img_p])
+
+    with gr.Tab("Text mode"):
+        # Images
+        with gr.Row(variant="panel"):
+            with gr.Column(scale=1):
+                cond_img_t.render()
+
+            with gr.Column(scale=1):
+                segm_img_t.render()
+
+        # Submit & Clear
+        with gr.Row():
+            with gr.Column():
+                with gr.Row():
+                    with gr.Column():
+                        contour_check = gr.Checkbox(value=True, label='withContours', info='draw the edges of the masks')
+                        text_box_t = gr.Textbox(label="text prompt", value="dog,cat")
+
+                    with gr.Column():
+                        segment_btn_t = gr.Button("Segment with text", variant='primary')
+                        clear_btn_t = gr.Button("Clear", variant="secondary")
+
+                gr.Markdown("Try some of the examples below")
+                gr.Examples(examples=examples,
+                            inputs=[cond_img_t],
+                            examples_per_page=18)
+
+            with gr.Column():
+                with gr.Accordion("Advanced options", open=True):
+                    conf_threshold_t = gr.Slider(0.0, 0.9, 0.8, step=0.05, label='clip threshold', info='object confidence threshold of clip')
+                    box_threshold_t = gr.Slider(0.0, 0.9, 0.5, step=0.05, label='box threshold', info='box nms threshold')
+                    crop_n_layers_t = gr.Slider(0, 3, 0, step=1, label='crop_n_layers', info='crop_n_layers of auto generator')
+                    crop_threshold_t = gr.Slider(0.0, 0.9, 0.5, step=0.05, label='crop threshold', info='crop nms threshold')
+
+                # Description
+                gr.Markdown(description_e)
+    segment_btn_t.click(text_segment_sementic,
+                        inputs=[
+                            cond_img_t,
+                            text_box_t,
+                            conf_threshold_t,
+                            box_threshold_t,
+                            crop_n_layers_t,
+                            crop_threshold_t
+                        ],
+                        outputs=[segm_img_t])
+
+    with gr.Tab("Box mode"):
+        # Images
+        with gr.Row(variant="panel"):
+            with gr.Column(scale=1):
+                cond_img_b.render()
+
+            with gr.Column(scale=1):
+                segm_img_b.render()
+
+        # Submit & Clear
+        with gr.Row():
+            with gr.Column():
+                with gr.Row():
+                    with gr.Column():
+                        contour_check = gr.Checkbox(value=True, label='withContours', info='draw the edges of the masks')
+                        text_box_b = gr.Textbox(label="vocabulary", value="dog,cat")
+                    with gr.Column():
+                        segment_btn_b = gr.Button("Segment with box", variant='primary')
+                        clear_btn_b = gr.Button("Clear", variant="secondary")
+
+                gr.Markdown("Try some of the examples below")
+                gr.Examples(examples=examples,
+                            inputs=[cond_img_t],
+                            examples_per_page=18)
+
+            with gr.Column():
+                # Description
+                gr.Markdown(description_b)
+    
+    segment_btn_b.click(segment_sementic,
+                        inputs=[
+                            cond_img_b,
+                            text_box_b,
+                        ],
+                        outputs=[segm_img_b])
+
+    def clear():
+        return None, None, None
+    
+    def clear_text():
+        return None, None, None
+
+    clear_btn_p.click(clear, outputs=[cond_img_p, segm_img_p, text_box_p])
+    clear_btn_t.click(clear_text, outputs=[cond_img_t, segm_img_t, text_box_t])
+    clear_btn_b.click(clear_text, outputs=[cond_img_b, segm_img_b, text_box_b])
+
+demo.queue()
+demo.launch()

regionspot/__init__.py

@@ -0,0 +1,10 @@
+from .config import add_regionspot_config
+from .detector import RegionSpot
+from .data.dataset_mapper import RegionSpotDatasetMapper
+from .test_time_augmentation import RegionSpotWithTTA
+from .build import *
+from .data.custom_dataset_dataloader import *
+from .predictor import RegionSpot_Predictor
+from .automatic_mask_generator import SamAutomaticMaskGenerator
+
+

regionspot/automatic_mask_generator.py

@@ -0,0 +1,632 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import numpy as np
+import torch
+from torchvision.ops.boxes import batched_nms, box_area  # type: ignore
+import json
+
+from typing import Any, Dict, List, Optional, Tuple
+
+from .modeling.segment_anything.utils.transforms import ResizeLongestSide
+# from .modeling import Sam
+# from .predictor import SamPredictor
+from .predictor import RegionSpot_Predictor
+from .modeling.segment_anything.utils.amg import (
+    MaskData,
+    area_from_rle,
+    batch_iterator,
+    batched_mask_to_box,
+    box_xyxy_to_xywh,
+    build_all_layer_point_grids,
+    calculate_stability_score,
+    coco_encode_rle,
+    generate_crop_boxes,
+    is_box_near_crop_edge,
+    mask_to_rle_pytorch,
+    remove_small_regions,
+    rle_to_mask,
+    uncrop_boxes_xyxy,
+    uncrop_masks,
+    uncrop_points,
+)
+
+
+class SamAutomaticMaskGenerator:
+    def __init__(
+        self,
+        model,
+        points_per_side: Optional[int] = 32,
+        points_per_batch: int = 64,
+        pred_iou_thresh: float = 0.88,
+        stability_score_thresh: float = 0.95,
+        stability_score_offset: float = 1.0,
+        box_nms_thresh: float = 0.7,
+        crop_n_layers: int = 0,
+        crop_nms_thresh: float = 0.7,
+        crop_overlap_ratio: float = 512 / 1500,
+        crop_n_points_downscale_factor: int = 1,
+        point_grids: Optional[List[np.ndarray]] = None,
+        min_mask_region_area: int = 0,
+        output_mode: str = "binary_mask",
+        crop_thresh = 0.0,
+        box_thresh = 0.6,
+        mask_threshold = 0.0,
+    ) -> None:
+        """
+        Using a SAM model, generates masks for the entire image.
+        Generates a grid of point prompts over the image, then filters
+        low quality and duplicate masks. The default settings are chosen
+        for SAM with a ViT-H backbone.
+
+        Arguments:
+          model (Sam): The SAM model to use for mask prediction.
+          points_per_side (int or None): The number of points to be sampled
+            along one side of the image. The total number of points is
+            points_per_side**2. If None, 'point_grids' must provide explicit
+            point sampling.
+          points_per_batch (int): Sets the number of points run simultaneously
+            by the model. Higher numbers may be faster but use more GPU memory.
+          pred_iou_thresh (float): A filtering threshold in [0,1], using the
+            model's predicted mask quality.
+          stability_score_thresh (float): A filtering threshold in [0,1], using
+            the stability of the mask under changes to the cutoff used to binarize
+            the model's mask predictions.
+          stability_score_offset (float): The amount to shift the cutoff when
+            calculated the stability score.
+          box_nms_thresh (float): The box IoU cutoff used by non-maximal
+            suppression to filter duplicate masks.
+          crop_n_layers (int): If >0, mask prediction will be run again on
+            crops of the image. Sets the number of layers to run, where each
+            layer has 2**i_layer number of image crops.
+          crop_nms_thresh (float): The box IoU cutoff used by non-maximal
+            suppression to filter duplicate masks between different crops.
+          crop_overlap_ratio (float): Sets the degree to which crops overlap.
+            In the first crop layer, crops will overlap by this fraction of
+            the image length. Later layers with more crops scale down this overlap.
+          crop_n_points_downscale_factor (int): The number of points-per-side
+            sampled in layer n is scaled down by crop_n_points_downscale_factor**n.
+          point_grids (list(np.ndarray) or None): A list over explicit grids
+            of points used for sampling, normalized to [0,1]. The nth grid in the
+            list is used in the nth crop layer. Exclusive with points_per_side.
+          min_mask_region_area (int): If >0, postprocessing will be applied
+            to remove disconnected regions and holes in masks with area smaller
+            than min_mask_region_area. Requires opencv.
+          output_mode (str): The form masks are returned in. Can be 'binary_mask',
+            'uncompressed_rle', or 'coco_rle'. 'coco_rle' requires pycocotools.
+            For large resolutions, 'binary_mask' may consume large amounts of
+            memory.
+        """
+
+        assert (points_per_side is None) != (
+            point_grids is None
+        ), "Exactly one of points_per_side or point_grid must be provided."
+        if points_per_side is not None:
+            self.point_grids = build_all_layer_point_grids(
+                points_per_side,
+                crop_n_layers,
+                crop_n_points_downscale_factor,
+            )
+        elif point_grids is not None:
+            self.point_grids = point_grids
+        else:
+            raise ValueError("Can't have both points_per_side and point_grid be None.")
+
+        assert output_mode in [
+            "binary_mask",
+            "uncompressed_rle",
+            "coco_rle",
+        ], f"Unknown output_mode {output_mode}."
+        if output_mode == "coco_rle":
+            from pycocotools import mask as mask_utils  # type: ignore # noqa: F401
+
+        if min_mask_region_area > 0:
+            import cv2  # type: ignore # noqa: F401
+        
+#         self.sam_clip = model
+#         self.model = self.sam_clip.sam    
+        self.predictor = RegionSpot_Predictor(model)
+        self.points_per_batch = points_per_batch
+        self.pred_iou_thresh = pred_iou_thresh
+        self.stability_score_thresh = stability_score_thresh
+        self.stability_score_offset = stability_score_offset
+        self.box_nms_thresh = box_nms_thresh
+        self.crop_n_layers = crop_n_layers
+        self.crop_nms_thresh = crop_nms_thresh
+        self.crop_overlap_ratio = crop_overlap_ratio
+        self.crop_n_points_downscale_factor = crop_n_points_downscale_factor
+        self.min_mask_region_area = min_mask_region_area
+        self.output_mode = output_mode
+        self.crop_thresh = crop_thresh
+        self.box_thresh = box_thresh
+        self.mask_threshold = mask_threshold
+
+    @torch.no_grad()
+    def generate(self, image: np.ndarray) -> List[Dict[str, Any]]:
+        """
+        Generates masks for the given image.
+
+        Arguments:
+          image (np.ndarray): The image to generate masks for, in HWC uint8 format.
+
+        Returns:
+           list(dict(str, any)): A list over records for masks. Each record is
+             a dict containing the following keys:
+               segmentation (dict(str, any) or np.ndarray): The mask. If
+                 output_mode='binary_mask', is an array of shape HW. Otherwise,
+                 is a dictionary containing the RLE.
+               bbox (list(float)): The box around the mask, in XYWH format.
+               area (int): The area in pixels of the mask.
+               predicted_iou (float): The model's own prediction of the mask's
+                 quality. This is filtered by the pred_iou_thresh parameter.
+               point_coords (list(list(float))): The point coordinates input
+                 to the model to generate this mask.
+               stability_score (float): A measure of the mask's quality. This
+                 is filtered on using the stability_score_thresh parameter.
+               crop_box (list(float)): The crop of the image used to generate
+                 the mask, given in XYWH format.
+        """
+
+        # Generate masks
+        mask_data = self._generate_masks(image)
+        # Filter small disconnected regions and holes in masks
+        if self.min_mask_region_area > 0:
+            mask_data = self.postprocess_small_regions(
+                mask_data,
+                self.min_mask_region_area,
+                max(self.box_nms_thresh, self.crop_nms_thresh),
+            )
+            
+            
+#         transform = ResizeLongestSide(self.model.image_encoder.img_size)
+        self.predictor.set_image(image,clip_input_size=336)
+        total_data = MaskData()
+        total_data["pred_class"]=[]
+        
+        maxvalue_box = 0
+        
+        for box in mask_data["boxes"]:
+            box = self.predictor.transform.apply_boxes(box, self.predictor.original_size)
+            box_torch = torch.as_tensor(box, dtype=torch.float, device=self.predictor.device)
+            box_torch = box_torch[None, :]
+            masks, iou_preds, _, max_values, max_index = self.predictor.predict_torch(
+                point_coords=None,
+                point_labels=None,
+                boxes=box_torch,
+                mask_input=None,
+                multimask_output=False,
+                mask_threshold = self.mask_threshold,
+            )
+            bmax_values = max_values.detach().cpu().numpy()
+            bmax_index = max_index.detach().cpu().numpy()
+            
+            pred_class = []
+            for i in range(bmax_index.shape[0]):
+                if bmax_values[i] > self.box_thresh:
+                    pred_class.append(bmax_index[i])
+                else:
+                    pred_class.append(-1)
+        # Serialize predictions and store in MaskData
+            data = MaskData(
+                masks=masks.flatten(0, 1),
+                iou_preds=iou_preds.flatten(0, 1),
+                pred_class=pred_class,
+            )
+            data["boxes"] = batched_mask_to_box(data["masks"])
+            data["rles"] = mask_to_rle_pytorch(data["masks"])
+            del data["masks"]
+            total_data.cat(data)
+        if total_data["pred_class"]==[]:
+            return False
+        if total_data["pred_class"]!=[]:
+            keep_mask= []
+            for i in total_data["pred_class"]:
+                if i != -1:
+                    keep_mask.append(True)
+                else:
+                    keep_mask.append(False)
+
+            keep_mask = torch.tensor(keep_mask)
+            total_data.filter(keep_mask)
+        mask_data = total_data
+
+        
+        # Encode masks
+        if self.output_mode == "coco_rle":
+            mask_data["segmentations"] = [coco_encode_rle(rle) for rle in mask_data["rles"]]
+        elif self.output_mode == "binary_mask":
+            mask_data["segmentations"] = [rle_to_mask(rle) for rle in mask_data["rles"]]
+        else:
+            mask_data["segmentations"] = mask_data["rles"]
+
+        # Write mask records
+        curr_anns = []
+        
+        for idx in range(len(mask_data["segmentations"])):
+            
+            ann = {
+                "segmentation": mask_data["segmentations"][idx],
+                "area": area_from_rle(mask_data["rles"][idx]),
+                "bbox": box_xyxy_to_xywh(mask_data["boxes"][idx]).tolist(),
+                "predicted_iou": mask_data["iou_preds"][idx].item(),
+                "pred_class": mask_data["pred_class"][idx],
+            }
+            curr_anns.append(ann)
+
+        return curr_anns
+
+    
+    
+###################
+    @torch.no_grad()
+    def generate_point(self, image: np.ndarray, point:np.ndarray, label:np.ndarray) -> List[Dict[str, Any]]:
+        """
+        Generates masks for the given image.
+
+        Arguments:
+          image (np.ndarray): The image to generate masks for, in HWC uint8 format.
+
+        Returns:
+           list(dict(str, any)): A list over records for masks. Each record is
+             a dict containing the following keys:
+               segmentation (dict(str, any) or np.ndarray): The mask. If
+                 output_mode='binary_mask', is an array of shape HW. Otherwise,
+                 is a dictionary containing the RLE.
+               bbox (list(float)): The box around the mask, in XYWH format.
+               area (int): The area in pixels of the mask.
+               predicted_iou (float): The model's own prediction of the mask's
+                 quality. This is filtered by the pred_iou_thresh parameter.
+               point_coords (list(list(float))): The point coordinates input
+                 to the model to generate this mask.
+               stability_score (float): A measure of the mask's quality. This
+                 is filtered on using the stability_score_thresh parameter.
+               crop_box (list(float)): The crop of the image used to generate
+                 the mask, given in XYWH format.
+        """
+        
+        
+        # Generate masks
+        mask_data = self._generate_masks_point(image, point, label)
+        # Filter small disconnected regions and holes in masks
+        if self.min_mask_region_area > 0:
+            mask_data = self.postprocess_small_regions(
+                mask_data,
+                self.min_mask_region_area,
+                max(self.box_nms_thresh, self.crop_nms_thresh),
+            )         
+#         transform = ResizeLongestSide(self.model.image_encoder.img_size)
+        self.predictor.set_image(image,clip_input_size=336)
+        total_data = MaskData()
+        total_data["pred_class"]=[]
+        
+        maxvalue_box = 0
+        
+        for box in mask_data["boxes"]:
+            box = self.predictor.transform.apply_boxes(box, self.predictor.original_size)
+            box_torch = torch.as_tensor(box, dtype=torch.float, device=self.predictor.device)
+            box_torch = box_torch[None, :]
+            masks, iou_preds, _, max_values, max_index = self.predictor.predict_torch(
+                point_coords=None,
+                point_labels=None,
+                boxes=box_torch,
+                mask_input=None,
+                multimask_output=False,
+                mask_threshold = self.mask_threshold,
+            )
+            bmax_values = max_values.detach().cpu().numpy()
+            bmax_index = max_index.detach().cpu().numpy()
+            
+            pred_class = []
+            maxV = []
+            for i in range(bmax_index.shape[0]):
+                if bmax_values[i] > self.box_thresh:
+                    pred_class.append(bmax_index[i])
+                else:
+                    pred_class.append(-1)
+            for i in range(bmax_index.shape[0]):
+                maxV.append(bmax_values[i])
+        # Serialize predictions and store in MaskData
+            data = MaskData(
+                masks=masks.flatten(0, 1),
+                iou_preds=iou_preds.flatten(0, 1),
+                pred_class=pred_class,
+                maxValue = maxV,
+            )
+            data["boxes"] = batched_mask_to_box(data["masks"])
+            data["rles"] = mask_to_rle_pytorch(data["masks"])
+            del data["masks"]
+            total_data.cat(data)
+            
+        if total_data["pred_class"]==[]:
+            return False
+        if total_data["maxValue"]!=[]:
+            keep_mask= []
+            for i in total_data["maxValue"]:
+                if i != max(total_data["maxValue"]):
+                    keep_mask.append(False)
+                else:
+                    keep_mask.append(True)
+            keep_mask = torch.tensor(keep_mask)
+            total_data.filter(keep_mask)
+        mask_data = total_data
+
+        
+        # Encode masks
+        if self.output_mode == "coco_rle":
+            mask_data["segmentations"] = [coco_encode_rle(rle) for rle in mask_data["rles"]]
+        elif self.output_mode == "binary_mask":
+            mask_data["segmentations"] = [rle_to_mask(rle) for rle in mask_data["rles"]]
+        else:
+            mask_data["segmentations"] = mask_data["rles"]
+
+        # Write mask records
+        curr_anns = []
+        
+        for idx in range(len(mask_data["segmentations"])):
+            
+            ann = {
+                "segmentation": mask_data["segmentations"][idx],
+                "area": area_from_rle(mask_data["rles"][idx]),
+                "bbox": box_xyxy_to_xywh(mask_data["boxes"][idx]).tolist(),
+                "predicted_iou": mask_data["iou_preds"][idx].item(),
+                "pred_class": mask_data["pred_class"][idx],
+            }
+            curr_anns.append(ann)
+
+        return curr_anns
+    
+    
+    def _generate_masks_point(self, image: np.ndarray, point:np.ndarray, label:np.ndarray) -> MaskData:
+        self.predictor.set_image(image,clip_input_size=336)
+        orig_size = image.shape[:2]
+
+        point = np.array(point)
+        transformed_points = self.predictor.transform.apply_coords(point, orig_size)
+        in_points = torch.as_tensor(transformed_points, device=self.predictor.device).reshape(-1,2)
+        in_labels = torch.as_tensor(label, device=self.predictor.device)
+#         self.predictor.set_image(image,clip_input_size=336)
+        masks, iou_preds, _, max_values, max_index = self.predictor.predict_torch(
+            in_points[:, None, :],
+            in_labels[:, None],
+            multimask_output=True,
+            return_logits=True,
+            mask_threshold = self.mask_threshold,
+        )
+        
+        max_values = max_values.detach().cpu().numpy()
+        max_index = max_index.detach().cpu().numpy()
+        pred_class = []
+        for i in range(max_index.shape[0]):
+            if max_values[i] > self.crop_thresh:                      
+                pred_class.append(max_index[i])
+            else:
+                pred_class.append(-1)
+
+        # Serialize predictions and store in MaskData
+        data = MaskData(
+            masks=masks.flatten(0, 1),
+            iou_preds=iou_preds.flatten(0, 1),
+            pred_class=pred_class,
+#             points=torch.as_tensor(points.repeat(masks.shape[1], axis=0)),
+        )
+        del masks
+        # Filter by predicted IoU
+#         if self.pred_iou_thresh > 0.0:
+#             keep_mask = data["iou_preds"] > self.pred_iou_thresh
+#             data.filter(keep_mask)
+        if data["pred_class"]!=[]:
+            
+            keep_mask= []
+            for i in data["pred_class"]:
+                if i != -1:
+                    keep_mask.append(True)
+                else:
+                    keep_mask.append(False)
+    
+            keep_mask = torch.tensor(keep_mask)
+            data.filter(keep_mask)
+
+        # Threshold masks and calculate boxes
+        data["masks"] = data["masks"] > self.predictor.model.mask_threshold
+        data["boxes"] = batched_mask_to_box(data["masks"])
+        data["rles"] = mask_to_rle_pytorch(data["masks"])
+        
+        del data["masks"]
+        
+        keep_by_nms = batched_nms(
+            data["boxes"].float(),
+            data["iou_preds"],
+            torch.zeros_like(data["boxes"][:, 0]),  # categories
+            iou_threshold=self.box_nms_thresh,
+        )
+        data.filter(keep_by_nms)
+        
+        data.to_numpy()
+        return data  
+    
+    def _generate_masks(self, image: np.ndarray) -> MaskData:
+        orig_size = image.shape[:2]
+        crop_boxes, layer_idxs = generate_crop_boxes(
+            orig_size, self.crop_n_layers, self.crop_overlap_ratio
+        )
+        # Iterate over image crops
+        data = MaskData()
+        for crop_box, layer_idx in zip(crop_boxes, layer_idxs):
+            crop_data = self._process_crop(image, crop_box, layer_idx, orig_size)
+            data.cat(crop_data)
+
+        # Remove duplicate masks between crops
+        if len(crop_boxes) > 1:
+            # Prefer masks from smaller crops
+            scores = 1 / box_area(data["crop_boxes"])
+            scores = scores.to(data["boxes"].device)
+            keep_by_nms = batched_nms(
+                data["boxes"].float(),
+                scores,
+                torch.zeros_like(data["boxes"][:, 0]),  # categories
+                iou_threshold=self.crop_nms_thresh,
+            )
+            data.filter(keep_by_nms)
+        data.to_numpy()
+        return data
+
+    def _process_crop(
+        self,
+        image: np.ndarray,
+        crop_box: List[int],
+        crop_layer_idx: int,
+        orig_size: Tuple[int, ...],
+    ) -> MaskData:
+        # Crop the image and calculate embeddings
+        x0, y0, x1, y1 = crop_box
+        cropped_im = image[y0:y1, x0:x1, :]
+        cropped_im_size = cropped_im.shape[:2]
+        self.predictor.set_image(cropped_im,clip_input_size=336)
+
+        # Get points for this crop
+        points_scale = np.array(cropped_im_size)[None, ::-1]
+        points_for_image = self.point_grids[crop_layer_idx] * points_scale
+
+        # Generate masks for this crop in batches
+        data = MaskData()
+        for (points,) in batch_iterator(self.points_per_batch, points_for_image):
+            batch_data = self._process_batch(points, cropped_im_size, crop_box, orig_size)
+            data.cat(batch_data)
+            del batch_data
+        self.predictor.reset_image()
+
+        # Remove duplicates within this crop.
+        keep_by_nms = batched_nms(
+            data["boxes"].float(),
+            data["iou_preds"],
+            torch.zeros_like(data["boxes"][:, 0]),  # categories
+            iou_threshold=self.box_nms_thresh,
+        )
+        data.filter(keep_by_nms)
+        # Return to the original image frame
+        data["boxes"] = uncrop_boxes_xyxy(data["boxes"], crop_box)
+        data["points"] = uncrop_points(data["points"], crop_box)
+        data["crop_boxes"] = torch.tensor([crop_box for _ in range(len(data["rles"]))])
+
+        return data
+
+    def _process_batch(
+        self,
+        points: np.ndarray,
+        im_size: Tuple[int, ...],
+        crop_box: List[int],
+        orig_size: Tuple[int, ...],
+    ) -> MaskData:
+        orig_h, orig_w = orig_size
+
+        # Run model on this batch
+        transformed_points = self.predictor.transform.apply_coords(points, im_size)
+        in_points = torch.as_tensor(transformed_points, device=self.predictor.device)
+        in_labels = torch.ones(in_points.shape[0], dtype=torch.int, device=in_points.device)
+#         import ipdb; ipdb.set_trace()
+        masks, iou_preds, _, max_values, max_index = self.predictor.predict_torch(
+            in_points[:, None, :],
+            in_labels[:, None],
+            multimask_output=True,
+            return_logits=True,
+            mask_threshold = self.mask_threshold,
+        )
+        max_values = max_values.detach().cpu().numpy()
+        max_index = max_index.detach().cpu().numpy()
+        pred_class = []
+        for i in range(max_index.shape[0]):
+            if max_values[i] > self.crop_thresh:                    
+                pred_class.append(max_index[i])
+            else:
+                pred_class.append(-1)
+
+        # Serialize predictions and store in MaskData
+        data = MaskData(
+            masks=masks.flatten(0, 1),
+            iou_preds=iou_preds.flatten(0, 1),
+            pred_class=pred_class,
+            points=torch.as_tensor(points.repeat(masks.shape[1], axis=0)),
+        )
+        del masks
+        
+        if data["pred_class"]!=[]:
+            
+            keep_mask= []
+            for i in data["pred_class"]:
+                if i != -1:
+                    keep_mask.append(True)
+                else:
+                    keep_mask.append(False)
+    
+            keep_mask = torch.tensor(keep_mask)
+            data.filter(keep_mask)
+
+        # Threshold masks and calculate boxes
+        data["masks"] = data["masks"] > self.predictor.model.mask_threshold
+        data["boxes"] = batched_mask_to_box(data["masks"])
+
+        # Filter boxes that touch crop boundaries
+        keep_mask = ~is_box_near_crop_edge(data["boxes"], crop_box, [0, 0, orig_w, orig_h])
+        if not torch.all(keep_mask):
+            data.filter(keep_mask)
+
+        # Compress to RLE
+        data["masks"] = uncrop_masks(data["masks"], crop_box, orig_h, orig_w)
+        data["rles"] = mask_to_rle_pytorch(data["masks"])
+        del data["masks"]
+
+        return data
+
+    @staticmethod
+    def postprocess_small_regions(
+        mask_data: MaskData, min_area: int, nms_thresh: float
+    ) -> MaskData:
+        """
+        Removes small disconnected regions and holes in masks, then reruns
+        box NMS to remove any new duplicates.
+
+        Edits mask_data in place.
+
+        Requires open-cv as a dependency.
+        """
+        if len(mask_data["rles"]) == 0:
+            return mask_data
+
+        # Filter small disconnected regions and holes
+        new_masks = []
+        scores = []
+        for rle in mask_data["rles"]:
+            mask = rle_to_mask(rle)
+
+            mask, changed = remove_small_regions(mask, min_area, mode="holes")
+            unchanged = not changed
+            mask, changed = remove_small_regions(mask, min_area, mode="islands")
+            unchanged = unchanged and not changed
+
+            new_masks.append(torch.as_tensor(mask).unsqueeze(0))
+            # Give score=0 to changed masks and score=1 to unchanged masks
+            # so NMS will prefer ones that didn't need postprocessing
+            scores.append(float(unchanged))
+
+        # Recalculate boxes and remove any new duplicates
+        masks = torch.cat(new_masks, dim=0)
+        boxes = batched_mask_to_box(masks)
+        keep_by_nms = batched_nms(
+            boxes.float(),
+            torch.as_tensor(scores),
+            torch.zeros_like(boxes[:, 0]),  # categories
+            iou_threshold=nms_thresh,
+        )
+
+        # Only recalculate RLEs for masks that have changed
+        for i_mask in keep_by_nms:
+            if scores[i_mask] == 0.0:
+                mask_torch = masks[i_mask].unsqueeze(0)
+                mask_data["rles"][i_mask] = mask_to_rle_pytorch(mask_torch)[0]
+                mask_data["boxes"][i_mask] = boxes[i_mask]  # update res directly
+        mask_data.filter(keep_by_nms)
+
+        return mask_data

regionspot/build.py

@@ -0,0 +1,307 @@
+import itertools
+import logging
+import torch.utils.data
+
+from detectron2.config import CfgNode, configurable
+from detectron2.data.build import (
+    build_batch_data_loader,
+    load_proposals_into_dataset,
+    trivial_batch_collator,
+)
+from detectron2.data.catalog import DatasetCatalog
+from detectron2.data.common import DatasetFromList, MapDataset
+from detectron2.data.dataset_mapper import DatasetMapper
+from detectron2.data.samplers import InferenceSampler, TrainingSampler
+from detectron2.utils.comm import get_world_size
+
+from torch.utils.data.sampler import Sampler
+from collections import defaultdict
+from typing import Optional
+from detectron2.utils import comm
+
+
+def _compute_num_images_per_worker(cfg: CfgNode):
+    num_workers = get_world_size()
+    images_per_batch = cfg.SOLVER.IMS_PER_BATCH
+    assert (
+        images_per_batch % num_workers == 0
+    ), "SOLVER.IMS_PER_BATCH ({}) must be divisible by the number of workers ({}).".format(
+        images_per_batch, num_workers
+    )
+    assert (
+        images_per_batch >= num_workers
+    ), "SOLVER.IMS_PER_BATCH ({}) must be larger than the number of workers ({}).".format(
+        images_per_batch, num_workers
+    )
+    images_per_worker = images_per_batch // num_workers
+    return images_per_worker
+
+
+def filter_images_with_only_crowd_annotations(dataset_dicts, dataset_names):
+    """
+    Filter out images with none annotations or only crowd annotations
+    (i.e., images without non-crowd annotations).
+    A common training-time preprocessing on COCO dataset.
+
+    Args:
+        dataset_dicts (list[dict]): annotations in Detectron2 Dataset format.
+
+    Returns:
+        list[dict]: the same format, but filtered.
+    """
+    num_before = len(dataset_dicts)
+
+    def valid(anns):
+        for ann in anns:
+            if isinstance(ann, list):
+                for instance in ann:
+                    if instance.get("iscrowd", 0) == 0:
+                        return True
+            else:
+                if ann.get("iscrowd", 0) == 0:
+                    return True
+        return False
+
+    dataset_dicts = [x for x in dataset_dicts if valid(x["annotations"])]
+    num_after = len(dataset_dicts)
+    logger = logging.getLogger(__name__)
+    logger.info(
+        "Removed {} images with no usable annotations. {} images left.".format(
+            num_before - num_after, num_after
+        )
+    )
+    return dataset_dicts
+
+
+def get_detection_dataset_dicts(
+    dataset_names, filter_empty=True, proposal_files=None
+):
+    """
+    Load and prepare dataset dicts for instance detection/segmentation and semantic segmentation.
+
+    Args:
+        dataset_names (str or list[str]): a dataset name or a list of dataset names
+        filter_empty (bool): whether to filter out images without instance annotations
+        proposal_files (list[str]): if given, a list of object proposal files
+            that match each dataset in `dataset_names`.
+
+    Returns:
+        list[dict]: a list of dicts following the standard dataset dict format.
+    """
+    if isinstance(dataset_names, str):
+        dataset_names = [dataset_names]
+    assert len(dataset_names)
+    dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in dataset_names]
+    for dataset_name, dicts in zip(dataset_names, dataset_dicts):
+        assert len(dicts), "Dataset '{}' is empty!".format(dataset_name)
+
+    if proposal_files is not None:
+        assert len(dataset_names) == len(proposal_files)
+        # load precomputed proposals from proposal files
+        dataset_dicts = [
+            load_proposals_into_dataset(dataset_i_dicts, proposal_file)
+            for dataset_i_dicts, proposal_file in zip(dataset_dicts, proposal_files)
+        ]
+
+    dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts))
+
+    has_instances = "annotations" in dataset_dicts[0]
+    if filter_empty and has_instances:
+        dataset_dicts = filter_images_with_only_crowd_annotations(dataset_dicts, dataset_names)
+
+    assert len(dataset_dicts), "No valid data found in {}.".format(",".join(dataset_names))
+    return dataset_dicts
+
+
+def _train_loader_from_config(cfg, mapper, *, dataset=None, sampler=None):
+    if dataset is None:
+        dataset = get_detection_dataset_dicts(
+            cfg.DATASETS.TRAIN,
+            filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS,
+            proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None,
+        )
+
+    if mapper is None:
+        mapper = DatasetMapper(cfg, True)
+
+    if sampler is None:
+        sampler_name = cfg.DATALOADER.SAMPLER_TRAIN
+        logger = logging.getLogger(__name__)
+        logger.info("Using training sampler {}".format(sampler_name))
+        if sampler_name == "TrainingSampler":
+            sampler = TrainingSampler(len(dataset))
+        elif sampler_name == "ClassAwareSampler":
+            sampler = ClassAwareSampler(dataset)
+
+    return {
+        "dataset": dataset,
+        "sampler": sampler,
+        "mapper": mapper,
+        "total_batch_size": cfg.SOLVER.IMS_PER_BATCH,
+        "aspect_ratio_grouping": cfg.DATALOADER.ASPECT_RATIO_GROUPING,
+        "num_workers": cfg.DATALOADER.NUM_WORKERS,
+        "use_mixup": True
+    }
+
+
+# TODO can allow dataset as an iterable or IterableDataset to make this function more general
+@configurable(from_config=_train_loader_from_config)
+def build_detection_train_loader(
+    dataset, *, mapper, sampler=None, total_batch_size, aspect_ratio_grouping=True, num_workers=0, 
+    use_mixup=False
+):
+    """
+    Build a dataloader for object detection with some default features.
+    This interface is experimental.
+
+    Args:
+        dataset (list or torch.utils.data.Dataset): a list of dataset dicts,
+            or a map-style pytorch dataset. They can be obtained by using
+            :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`.
+        mapper (callable): a callable which takes a sample (dict) from dataset and
+            returns the format to be consumed by the model.
+            When using cfg, the default choice is ``DatasetMapper(cfg, is_train=True)``.
+        sampler (torch.utils.data.sampler.Sampler or None): a sampler that
+            produces indices to be applied on ``dataset``.
+            Default to :class:`TrainingSampler`, which coordinates a random shuffle
+            sequence across all workers.
+        total_batch_size (int): total batch size across all workers. Batching
+            simply puts data into a list.
+        aspect_ratio_grouping (bool): whether to group images with similar
+            aspect ratio for efficiency. When enabled, it requires each
+            element in dataset be a dict with keys "width" and "height".
+        num_workers (int): number of parallel data loading workers
+
+    Returns:
+        torch.utils.data.DataLoader: a dataloader. Each output from it is a
+            ``list[mapped_element]`` of length ``total_batch_size / num_workers``,
+            where ``mapped_element`` is produced by the ``mapper``.
+    """
+    if isinstance(dataset, list):
+        dataset = DatasetFromList(dataset, copy=False)
+    if mapper is not None:
+        if use_mixup:
+            dataset = MapDatasetMixup(dataset, mapper)
+        else:
+            dataset = MapDataset(dataset, mapper)
+    if sampler is None:
+        sampler = TrainingSampler(len(dataset))
+    assert isinstance(sampler, torch.utils.data.sampler.Sampler)
+    return build_batch_data_loader(
+        dataset,
+        sampler,
+        total_batch_size,
+        aspect_ratio_grouping=aspect_ratio_grouping,
+        num_workers=num_workers,
+    )
+
+
+def _test_loader_from_config(cfg, dataset_name, mapper=None):
+    """
+    Uses the given `dataset_name` argument (instead of the names in cfg), because the
+    standard practice is to evaluate each test set individually (not combining them).
+    """
+    dataset = get_detection_dataset_dicts(
+        [dataset_name],
+        filter_empty=False,
+        proposal_files=[
+            cfg.DATASETS.PROPOSAL_FILES_TEST[list(cfg.DATASETS.TEST).index(dataset_name)]
+        ]
+        if cfg.MODEL.LOAD_PROPOSALS
+        else None,
+    )
+    if mapper is None:
+        mapper = DatasetMapper(cfg, False)
+    return {"dataset": dataset, "mapper": mapper, "num_workers": cfg.DATALOADER.NUM_WORKERS}
+
+
+@configurable(from_config=_test_loader_from_config)
+def build_detection_test_loader(dataset, *, mapper, num_workers=0):
+    """
+    Similar to `build_detection_train_loader`, but uses a batch size of 1.
+    This interface is experimental.
+
+    Args:
+        dataset (list or torch.utils.data.Dataset): a list of dataset dicts,
+            or a map-style pytorch dataset. They can be obtained by using
+            :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`.
+        mapper (callable): a callable which takes a sample (dict) from dataset
+           and returns the format to be consumed by the model.
+           When using cfg, the default choice is ``DatasetMapper(cfg, is_train=False)``.
+        num_workers (int): number of parallel data loading workers
+
+    Returns:
+        DataLoader: a torch DataLoader, that loads the given detection
+        dataset, with test-time transformation and batching.
+
+    Examples:
+    ::
+        data_loader = build_detection_test_loader(
+            DatasetRegistry.get("my_test"),
+            mapper=DatasetMapper(...))
+
+        # or, instantiate with a CfgNode:
+        data_loader = build_detection_test_loader(cfg, "my_test")
+    """
+    if isinstance(dataset, list):
+        dataset = DatasetFromList(dataset, copy=False)
+    if mapper is not None:
+        dataset = MapDataset(dataset, mapper)
+    sampler = InferenceSampler(len(dataset))
+    # Always use 1 image per worker during inference since this is the
+    # standard when reporting inference time in papers.
+    # batch_sampler = torch.utils.data.sampler.BatchSampler(sampler, 1, drop_last=False)
+    data_loader = torch.utils.data.DataLoader(
+        dataset,
+        batch_size=1,
+        sampler=sampler,
+        drop_last=False,
+        num_workers=num_workers,
+        collate_fn=trivial_batch_collator,
+    )
+    return data_loader
+
+
+class ClassAwareSampler(Sampler):
+    def __init__(self, dataset_dicts, seed: Optional[int] = None):
+        """
+        """
+        self._size = len(dataset_dicts)
+        assert self._size > 0
+        if seed is None:
+            seed = comm.shared_random_seed()
+        self._seed = int(seed)
+        
+        self._rank = comm.get_rank()
+        self._world_size = comm.get_world_size()
+        self.weights = self._get_class_balance_factor(dataset_dicts)
+
+
+    def __iter__(self):
+        start = self._rank
+        yield from itertools.islice(
+            self._infinite_indices(), start, None, self._world_size)
+
+
+    def _infinite_indices(self):
+        g = torch.Generator()
+        g.manual_seed(self._seed)
+        while True:
+            ids = torch.multinomial(
+                self.weights, self._size, generator=g, 
+                replacement=True)
+            yield from ids
+
+
+    def _get_class_balance_factor(self, dataset_dicts, l=1.):
+        ret = []
+        category_freq = defaultdict(int)
+        for dataset_dict in dataset_dicts:  # For each image (without repeats)
+            cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]}
+            for cat_id in cat_ids:
+                category_freq[cat_id] += 1
+        for i, dataset_dict in enumerate(dataset_dicts):
+            cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]}
+            ret.append(sum(
+                [1. / (category_freq[cat_id] ** l) for cat_id in cat_ids]))
+        return torch.tensor(ret).float()

regionspot/config.py

@@ -0,0 +1,39 @@
+from detectron2.config import CfgNode as CN
+
+
+def add_regionspot_config(cfg):
+    """
+    Add config for RegionSpot
+    """
+    cfg.MODEL.RegionSpot = CN()
+    cfg.MODEL.CLIP_TYPE = 'CLIP_400M_Large'
+    cfg.MODEL.CLIP_INPUT_SIZE = 224
+    # Inference
+    cfg.MODEL.TRAINING = True
+    cfg.MODEL.BOX_TYPE = 'GT'
+    
+    #Dataloder
+    cfg.DATALOADER.DATASET_RATIO = [1,1,1] # sample ratio
+    cfg.DATALOADER.USE_RFS = [False, False, False]
+    cfg.DATALOADER.MULTI_DATASET_GROUPING = True # Always true when multi-dataset is enabled
+    cfg.DATALOADER.DATASET_ANN = ['box', 'box', 'box'] # Annotation type of each dataset
+    cfg.DATALOADER.USE_DIFF_BS_SIZE = False # Use different batchsize for each dataset
+    cfg.DATALOADER.DATASET_BS = [8, 32] # Used when USE_DIFF_BS_SIZE is on
+   
+    
+
+    # Optimizer.
+    cfg.SOLVER.OPTIMIZER = "ADAMW"
+    cfg.SOLVER.BACKBONE_MULTIPLIER = 1.0
+
+    # TTA.
+    cfg.TEST.AUG.MIN_SIZES = (400, 500, 600, 640, 700, 900, 1000, 1100, 1200, 1300, 1400, 1800, 800)
+    cfg.TEST.AUG.CVPODS_TTA = True
+    cfg.TEST.AUG.SCALE_FILTER = True
+    cfg.TEST.AUG.SCALE_RANGES = ([96, 10000], [96, 10000], 
+                                 [64, 10000], [64, 10000],
+                                 [64, 10000], [0, 10000],
+                                 [0, 10000], [0, 256],
+                                 [0, 256], [0, 192],
+                                 [0, 192], [0, 96],
+                                 [0, 10000])

regionspot/data/__pycache__/v3det_categories.cpython-38.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ba3ef77b19c8f288db6ed9ac232384853c7d27413c35d662423586824480519
+size 1552733

regionspot/data/custom_dataset_dataloader.py

@@ -0,0 +1,331 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# Part of the code is from https://github.com/xingyizhou/UniDet/blob/master/projects/UniDet/unidet/data/multi_dataset_dataloader.py (Apache-2.0 License)
+import copy
+import logging
+import numpy as np
+import operator
+import torch
+import torch.utils.data
+import json
+from detectron2.utils.comm import get_world_size
+from detectron2.utils.logger import _log_api_usage, log_first_n
+
+from detectron2.config import configurable
+from detectron2.data import samplers
+from torch.utils.data.sampler import BatchSampler, Sampler
+from detectron2.data.common import DatasetFromList, MapDataset
+from detectron2.data.dataset_mapper import DatasetMapper
+from detectron2.data.build import get_detection_dataset_dicts, build_batch_data_loader
+from detectron2.data.samplers import TrainingSampler, RepeatFactorTrainingSampler
+from detectron2.data.build import worker_init_reset_seed, print_instances_class_histogram
+from detectron2.data.build import filter_images_with_only_crowd_annotations
+from detectron2.data.build import filter_images_with_few_keypoints
+from detectron2.data.build import check_metadata_consistency
+from detectron2.data.catalog import MetadataCatalog, DatasetCatalog
+from detectron2.utils import comm
+import itertools
+import math
+from collections import defaultdict
+from typing import Optional
+
+
+def _custom_train_loader_from_config(cfg, mapper=None, *, dataset=None, sampler=None):
+    sampler_name = cfg.DATALOADER.SAMPLER_TRAIN # "MultiDatasetSampler"
+    if 'MultiDataset' in sampler_name: # True
+        dataset_dicts = get_detection_dataset_dicts_with_source(
+            cfg.DATASETS.TRAIN,
+            filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS,
+            min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE
+            if cfg.MODEL.KEYPOINT_ON else 0,
+            proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None,
+        )
+    else: # False
+        dataset_dicts = get_detection_dataset_dicts(
+            cfg.DATASETS.TRAIN,
+            filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS,
+            min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE
+            if cfg.MODEL.KEYPOINT_ON else 0,
+            proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None,
+        )
+
+    if mapper is None: # False
+        mapper = DatasetMapper(cfg, True)
+
+    if sampler is not None:
+        pass
+    elif sampler_name == "TrainingSampler": # False
+        sampler = TrainingSampler(len(dataset))
+    elif sampler_name == "MultiDatasetSampler": # True
+        sampler = MultiDatasetSampler(
+            dataset_dicts,
+            dataset_ratio = cfg.DATALOADER.DATASET_RATIO,
+            use_rfs = cfg.DATALOADER.USE_RFS,
+            dataset_ann = cfg.DATALOADER.DATASET_ANN,
+            repeat_threshold = cfg.DATALOADER.REPEAT_THRESHOLD,
+        )
+    elif sampler_name == "RepeatFactorTrainingSampler": # False
+        repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency(
+            dataset_dicts, cfg.DATALOADER.REPEAT_THRESHOLD
+        )
+        sampler = RepeatFactorTrainingSampler(repeat_factors)
+    else:
+        raise ValueError("Unknown training sampler: {}".format(sampler_name))
+
+    return {
+        "dataset": dataset_dicts,
+        "sampler": sampler,
+        "mapper": mapper,
+        "total_batch_size": cfg.SOLVER.IMS_PER_BATCH, # 64
+        "aspect_ratio_grouping": cfg.DATALOADER.ASPECT_RATIO_GROUPING,
+        "num_workers": cfg.DATALOADER.NUM_WORKERS, # 8
+        'multi_dataset_grouping': cfg.DATALOADER.MULTI_DATASET_GROUPING, # True
+        'use_diff_bs_size': cfg.DATALOADER.USE_DIFF_BS_SIZE, # True
+        'dataset_bs': cfg.DATALOADER.DATASET_BS, # [8, 32]
+        'num_datasets': len(cfg.DATASETS.TRAIN) # 2
+    }
+
+
+@configurable(from_config=_custom_train_loader_from_config)
+def build_custom_train_loader(
+        dataset, *, mapper, sampler, 
+        total_batch_size=16, # 64
+        aspect_ratio_grouping=True, 
+        num_workers=0, # 8
+        num_datasets=1, # 2
+        multi_dataset_grouping=False, # True
+        use_diff_bs_size=False, # True
+        dataset_bs=[] # [8, 32]
+    ):
+    """
+    Modified from detectron2.data.build.build_custom_train_loader, but supports
+    different samplers
+    """
+    if isinstance(dataset, list):
+        dataset = DatasetFromList(dataset, copy=False)
+    if mapper is not None: # True
+        dataset = MapDataset(dataset, mapper)
+    if sampler is None: # False
+        sampler = TrainingSampler(len(dataset))
+    assert isinstance(sampler, torch.utils.data.sampler.Sampler)
+    if multi_dataset_grouping: # True
+        return build_multi_dataset_batch_data_loader(
+            use_diff_bs_size,
+            dataset_bs,
+            dataset,
+            sampler,
+            total_batch_size,
+            num_datasets=num_datasets,
+            num_workers=num_workers,
+        )
+    else: # False
+        return build_batch_data_loader(
+            dataset,
+            sampler,
+            total_batch_size,
+            aspect_ratio_grouping=aspect_ratio_grouping,
+            num_workers=num_workers,
+        )
+
+
+def build_multi_dataset_batch_data_loader(
+    use_diff_bs_size, dataset_bs,
+    dataset, sampler, total_batch_size, num_datasets, num_workers=0
+):
+    """
+    """
+    world_size = get_world_size()
+    assert (
+        total_batch_size > 0 and total_batch_size % world_size == 0
+    ), "Total batch size ({}) must be divisible by the number of gpus ({}).".format(
+        total_batch_size, world_size
+    )
+
+    batch_size = total_batch_size // world_size
+    data_loader = torch.utils.data.DataLoader(
+        dataset,
+        sampler=sampler,
+        num_workers=num_workers,
+        batch_sampler=None,
+        collate_fn=operator.itemgetter(0),  # don't batch, but yield individual elements
+        worker_init_fn=worker_init_reset_seed,
+    )  # yield individual mapped dict
+    if use_diff_bs_size:
+        return DIFFMDAspectRatioGroupedDataset(
+            data_loader, dataset_bs, num_datasets)
+    else:
+        return MDAspectRatioGroupedDataset(
+            data_loader, batch_size, num_datasets)
+
+
+def get_detection_dataset_dicts_with_source(
+    dataset_names, filter_empty=True, min_keypoints=0, proposal_files=None
+):
+    assert len(dataset_names)
+    dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in dataset_names]
+    for dataset_name, dicts in zip(dataset_names, dataset_dicts):
+        assert len(dicts), "Dataset '{}' is empty!".format(dataset_name)
+    
+    for source_id, (dataset_name, dicts) in \
+        enumerate(zip(dataset_names, dataset_dicts)):
+        assert len(dicts), "Dataset '{}' is empty!".format(dataset_name)
+        for d in dicts:
+            d['dataset_source'] = source_id # add "dataset_source" to original dict
+
+        if "annotations" in dicts[0]:
+            try:
+                class_names = MetadataCatalog.get(dataset_name).thing_classes
+                check_metadata_consistency("thing_classes", dataset_name)
+                print_instances_class_histogram(dicts, class_names)
+            except AttributeError:  # class names are not available for this dataset
+                pass
+
+    assert proposal_files is None
+
+    dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts)) # connect multiple iterable objects to one
+
+    has_instances = "annotations" in dataset_dicts[0]
+    if filter_empty and has_instances:
+        dataset_dicts = filter_images_with_only_crowd_annotations(dataset_dicts)
+    if min_keypoints > 0 and has_instances:
+        dataset_dicts = filter_images_with_few_keypoints(dataset_dicts, min_keypoints)
+
+    return dataset_dicts
+
+
+class MultiDatasetSampler(Sampler):
+    def __init__(
+        self, 
+        dataset_dicts, 
+        dataset_ratio,
+        use_rfs, # [True, False]
+        dataset_ann,
+        repeat_threshold=0.001,
+        seed: Optional[int] = None,
+        ):
+        """
+        """
+        sizes = [0 for _ in range(len(dataset_ratio))]
+        for d in dataset_dicts:
+            sizes[d['dataset_source']] += 1 # size of each dataset
+        print('dataset sizes', sizes)
+        self.sizes = sizes
+        assert len(dataset_ratio) == len(sizes), \
+            'length of dataset ratio {} should be equal to number if dataset {}'.format(
+                len(dataset_ratio), len(sizes)
+            )
+        if seed is None:
+            seed = comm.shared_random_seed() # seed shared across all GPUs
+        self._seed = int(seed)
+        self._rank = comm.get_rank()
+        self._world_size = comm.get_world_size()
+        
+        self.dataset_ids =  torch.tensor(
+            [d['dataset_source'] for d in dataset_dicts], dtype=torch.long)
+
+        dataset_weight = [torch.ones(s) * max(sizes) / s * r / sum(dataset_ratio) \
+            for i, (r, s) in enumerate(zip(dataset_ratio, sizes))]
+        dataset_weight = torch.cat(dataset_weight)
+        
+        rfs_factors = []
+        st = 0
+        for i, s in enumerate(sizes):
+            if use_rfs[i]:
+                if dataset_ann[i] == 'box':
+                    rfs_func = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency
+                else:
+                    rfs_func = repeat_factors_from_tag_frequency
+                rfs_factor = rfs_func(
+                    dataset_dicts[st: st + s],
+                    repeat_thresh=repeat_threshold)
+                rfs_factor = rfs_factor * (s / rfs_factor.sum())
+            else:
+                rfs_factor = torch.ones(s)
+            rfs_factors.append(rfs_factor)
+            st = st + s
+        rfs_factors = torch.cat(rfs_factors)
+
+        self.weights = dataset_weight * rfs_factors # weights for each element in the dataset_dict
+        self.sample_epoch_size = len(self.weights)
+
+    def __iter__(self):
+        start = self._rank
+        yield from itertools.islice(
+            self._infinite_indices(), start, None, self._world_size) # itertools.islice(iterable, start, stop[, step])
+
+
+    def _infinite_indices(self):
+        g = torch.Generator()
+        g.manual_seed(self._seed)
+        while True:
+            ids = torch.multinomial(
+                self.weights, self.sample_epoch_size, generator=g, 
+                replacement=True) # randomly sample according to the given weights
+            nums = [(self.dataset_ids[ids] == i).sum().int().item() \
+                for i in range(len(self.sizes))]
+            yield from ids
+
+
+class MDAspectRatioGroupedDataset(torch.utils.data.IterableDataset):
+    def __init__(self, dataset, batch_size, num_datasets):
+        """
+        """
+        self.dataset = dataset
+        self.batch_size = batch_size
+        self._buckets = [[] for _ in range(2 * num_datasets)] # there are (2 x num_datasets) types of data. For each dataset, there are two types: w>h or w<=h
+
+    def __iter__(self):
+        for d in self.dataset:
+            w, h = d["width"], d["height"]
+            aspect_ratio_bucket_id = 0 if w > h else 1
+            bucket_id = d['dataset_source'] * 2 + aspect_ratio_bucket_id
+            bucket = self._buckets[bucket_id]
+            bucket.append(d)
+            if len(bucket) == self.batch_size:
+                yield bucket[:]
+                del bucket[:]
+
+
+class DIFFMDAspectRatioGroupedDataset(torch.utils.data.IterableDataset):
+    def __init__(self, dataset, batch_sizes, num_datasets):
+        """
+        """
+        self.dataset = dataset
+        self.batch_sizes = batch_sizes
+        self._buckets = [[] for _ in range(2 * num_datasets)]
+
+    def __iter__(self):
+        for d in self.dataset:
+            w, h = d["width"], d["height"]
+            aspect_ratio_bucket_id = 0 if w > h else 1
+            bucket_id = d['dataset_source'] * 2 + aspect_ratio_bucket_id
+            bucket = self._buckets[bucket_id]
+            bucket.append(d)
+            if len(bucket) == self.batch_sizes[d['dataset_source']]: # allow different batchsizes
+                yield bucket[:]
+                del bucket[:]
+
+
+def repeat_factors_from_tag_frequency(dataset_dicts, repeat_thresh):
+    """
+    """
+    category_freq = defaultdict(int)
+    for dataset_dict in dataset_dicts:
+        cat_ids = dataset_dict['pos_category_ids']
+        for cat_id in cat_ids:
+            category_freq[cat_id] += 1
+    num_images = len(dataset_dicts)
+    for k, v in category_freq.items():
+        category_freq[k] = v / num_images
+
+    category_rep = {
+        cat_id: max(1.0, math.sqrt(repeat_thresh / cat_freq))
+        for cat_id, cat_freq in category_freq.items()
+    }
+
+    rep_factors = []
+    for dataset_dict in dataset_dicts:
+        cat_ids = dataset_dict['pos_category_ids']
+        rep_factor = max({category_rep[cat_id] for cat_id in cat_ids}, default=1.0)
+        rep_factors.append(rep_factor)
+
+    return torch.tensor(rep_factors, dtype=torch.float32)

regionspot/data/dataset_mapper.py

@@ -0,0 +1,140 @@
+# ========================================
+# Modified by Shoufa Chen
+# ========================================
+# Modified by Peize Sun, Rufeng Zhang
+# Contact: {sunpeize, cxrfzhang}@foxmail.com
+#
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import copy
+import logging
+import numpy as np
+import torch
+import os
+from detectron2.data import detection_utils as utils
+from detectron2.data import transforms as T
+
+
+__all__ = ["RegionSpotDatasetMapper"]
+
+
+def build_transform_gen(cfg, is_train):
+    """
+    Create a list of :class:`TransformGen` from config.
+    Returns:
+        list[TransformGen]
+    """
+    if is_train:
+        min_size = cfg.INPUT.MIN_SIZE_TRAIN
+        max_size = cfg.INPUT.MAX_SIZE_TRAIN
+        sample_style = cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING
+    else:
+        min_size = cfg.INPUT.MIN_SIZE_TEST
+        max_size = cfg.INPUT.MAX_SIZE_TEST
+        sample_style = "choice"
+    if sample_style == "range":
+        assert len(min_size) == 2, "more than 2 ({}) min_size(s) are provided for ranges".format(len(min_size))
+
+    logger = logging.getLogger(__name__)
+    tfm_gens = []
+    if is_train:
+        tfm_gens.append(T.RandomFlip())
+    # ResizeShortestEdge
+    tfm_gens.append(T.ResizeShortestEdge(min_size, max_size, sample_style))
+
+    if is_train:
+        logger.info("TransformGens used in training: " + str(tfm_gens))
+    return tfm_gens
+
+
+class RegionSpotDatasetMapper:
+    """
+    A callable which takes a dataset dict in Detectron2 Dataset format,
+    and map it into a format used by DiffusionDet.
+
+    The callable currently does the following:
+
+    1. Read the image from "file_name"
+    2. Applies geometric transforms to the image and annotation
+    3. Find and applies suitable cropping to the image and annotation
+    4. Prepare image and annotation to Tensors
+    """
+
+    def __init__(self, cfg, is_train=True):
+        if cfg.INPUT.CROP.ENABLED and is_train:
+            self.crop_gen = [
+                T.ResizeShortestEdge([400, 500, 600], sample_style="choice"),
+                T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE),
+            ]
+        else:
+            self.crop_gen = None
+
+        self.tfm_gens = build_transform_gen(cfg, is_train)
+        logging.getLogger(__name__).info(
+            "Full TransformGens used in training: {}, crop: {}".format(str(self.tfm_gens), str(self.crop_gen))
+        )
+
+        self.img_format = cfg.INPUT.FORMAT
+        self.is_train = is_train
+        # if self.is_train:
+        #     for dataset_name in cfg.DATASETS.TRAIN:
+        #         if dataset_name.startswith("coco"):
+        self.mask_tokens_dir = os.path.join('./datasets/datasets_mask_tokens_vit_b/')
+
+    def __call__(self, dataset_dict):
+        """
+        Args:
+            dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format.
+
+        Returns:
+            dict: a format that builtin models in detectron2 accept
+        """
+        dataset_dict = copy.deepcopy(dataset_dict)  # it will be modified by code below
+        image = utils.read_image(dataset_dict["file_name"], format=self.img_format)
+        # utils.check_image_size(dataset_dict, image)
+        #
+        #get mask token and responsed label
+        image_id = dataset_dict["image_id"]
+        dataset_name = dataset_dict["file_name"].split('/')[1]
+        #datasets/coco/train2017/000000566174.jpg
+        #read pth
+        pth_file = os.path.join(self.mask_tokens_dir, os.path.join(dataset_name, str(image_id)+'.pth'))
+        offline_token = torch.load(pth_file)
+        #
+        if self.crop_gen is None:
+            image, transforms = T.apply_transform_gens(self.tfm_gens, image)
+        else:
+            if np.random.rand() > 0.5:
+                image, transforms = T.apply_transform_gens(self.tfm_gens, image)
+            else:
+                image, transforms = T.apply_transform_gens(
+                    self.tfm_gens[:-1] + self.crop_gen + self.tfm_gens[-1:], image
+                )
+
+        image_shape = image.shape[:2]  # h, w
+
+        # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory,
+        # but not efficient on large generic data structures due to the use of pickle & mp.Queue.
+        # Therefore it's important to use torch.Tensor.
+        dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1)))
+        dataset_dict["dataset_name"] = dataset_name
+        dataset_dict["extra_info"] = offline_token
+        if not self.is_train:
+            # USER: Modify this if you want to keep them for some reason.
+            dataset_dict.pop("annotations", None)
+            return dataset_dict
+
+        if "annotations" in dataset_dict:
+            # USER: Modify this if you want to keep them for some reason.
+            for anno in dataset_dict["annotations"]:
+                anno.pop("segmentation", None)
+                anno.pop("keypoints", None)
+
+            # USER: Implement additional transformations if you have other types of data
+            annos = [
+                utils.transform_instance_annotations(obj, transforms, image_shape)
+                for obj in dataset_dict.pop("annotations")
+                if obj.get("iscrowd", 0) == 0
+            ]
+            instances = utils.annotations_to_instances(annos, image_shape)
+            dataset_dict["instances"] = utils.filter_empty_instances(instances)
+        return dataset_dict

regionspot/data/objects365.py

@@ -0,0 +1,391 @@
+from detectron2.data.datasets.register_coco import register_coco_instances
+import os
+
+categories = [
+{'id': 164, 'name': 'cutting/chopping board'} ,
+{'id': 49, 'name': 'tie'} ,
+{'id': 306, 'name': 'crosswalk sign'} ,
+{'id': 145, 'name': 'gun'} ,
+{'id': 14, 'name': 'street lights'} ,
+{'id': 223, 'name': 'bar soap'} ,
+{'id': 74, 'name': 'wild bird'} ,
+{'id': 219, 'name': 'ice cream'} ,
+{'id': 37, 'name': 'stool'} ,
+{'id': 25, 'name': 'storage box'} ,
+{'id': 153, 'name': 'giraffe'} ,
+{'id': 52, 'name': 'pen/pencil'} ,
+{'id': 61, 'name': 'high heels'} ,
+{'id': 340, 'name': 'mangosteen'} ,
+{'id': 22, 'name': 'bracelet'} ,
+{'id': 155, 'name': 'piano'} ,
+{'id': 162, 'name': 'vent'} ,
+{'id': 75, 'name': 'laptop'} ,
+{'id': 236, 'name': 'toaster'} ,
+{'id': 231, 'name': 'fire truck'} ,
+{'id': 42, 'name': 'basket'} ,
+{'id': 150, 'name': 'zebra'} ,
+{'id': 124, 'name': 'head phone'} ,
+{'id': 90, 'name': 'sheep'} ,
+{'id': 322, 'name': 'steak'} ,
+{'id': 39, 'name': 'couch'} ,
+{'id': 209, 'name': 'toothbrush'} ,
+{'id': 59, 'name': 'bicycle'} ,
+{'id': 336, 'name': 'red cabbage'} ,
+{'id': 228, 'name': 'golf ball'} ,
+{'id': 120, 'name': 'tomato'} ,
+{'id': 132, 'name': 'computer box'} ,
+{'id': 8, 'name': 'cup'} ,
+{'id': 183, 'name': 'basketball'} ,
+{'id': 298, 'name': 'butterfly'} ,
+{'id': 250, 'name': 'garlic'} ,
+{'id': 12, 'name': 'desk'} ,
+{'id': 141, 'name': 'microwave'} ,
+{'id': 171, 'name': 'strawberry'} ,
+{'id': 200, 'name': 'kettle'} ,
+{'id': 63, 'name': 'van'} ,
+{'id': 300, 'name': 'cheese'} ,
+{'id': 215, 'name': 'marker'} ,
+{'id': 100, 'name': 'blackboard/whiteboard'} ,
+{'id': 186, 'name': 'printer'} ,
+{'id': 333, 'name': 'bread/bun'} ,
+{'id': 243, 'name': 'penguin'} ,
+{'id': 364, 'name': 'iron'} ,
+{'id': 180, 'name': 'ladder'} ,
+{'id': 34, 'name': 'flag'} ,
+{'id': 78, 'name': 'cell phone'} ,
+{'id': 97, 'name': 'fan'} ,
+{'id': 224, 'name': 'scale'} ,
+{'id': 151, 'name': 'duck'} ,
+{'id': 319, 'name': 'flute'} ,
+{'id': 156, 'name': 'stop sign'} ,
+{'id': 290, 'name': 'rickshaw'} ,
+{'id': 128, 'name': 'sailboat'} ,
+{'id': 165, 'name': 'tennis racket'} ,
+{'id': 241, 'name': 'cigar'} ,
+{'id': 101, 'name': 'balloon'} ,
+{'id': 308, 'name': 'hair drier'} ,
+{'id': 167, 'name': 'skating and skiing shoes'} ,
+{'id': 237, 'name': 'helicopter'} ,
+{'id': 65, 'name': 'sink'} ,
+{'id': 129, 'name': 'tangerine'} ,
+{'id': 330, 'name': 'crab'} ,
+{'id': 320, 'name': 'measuring cup'} ,
+{'id': 260, 'name': 'fishing rod'} ,
+{'id': 346, 'name': 'saw'} ,
+{'id': 216, 'name': 'ship'} ,
+{'id': 46, 'name': 'coffee table'} ,
+{'id': 194, 'name': 'facial mask'} ,
+{'id': 281, 'name': 'stapler'} ,
+{'id': 118, 'name': 'refrigerator'} ,
+{'id': 40, 'name': 'belt'} ,
+{'id': 349, 'name': 'starfish'} ,
+{'id': 87, 'name': 'hanger'} ,
+{'id': 116, 'name': 'baseball glove'} ,
+{'id': 261, 'name': 'cherry'} ,
+{'id': 334, 'name': 'baozi'} ,
+{'id': 267, 'name': 'screwdriver'} ,
+{'id': 158, 'name': 'converter'} ,
+{'id': 335, 'name': 'lion'} ,
+{'id': 170, 'name': 'baseball'} ,
+{'id': 111, 'name': 'skis'} ,
+{'id': 136, 'name': 'broccoli'} ,
+{'id': 342, 'name': 'eraser'} ,
+{'id': 337, 'name': 'polar bear'} ,
+{'id': 139, 'name': 'shovel'} ,
+{'id': 193, 'name': 'extension cord'} ,
+{'id': 284, 'name': 'goldfish'} ,
+{'id': 174, 'name': 'pepper'} ,
+{'id': 138, 'name': 'stroller'} ,
+{'id': 328, 'name': 'yak'} ,
+{'id': 83, 'name': 'clock'} ,
+{'id': 235, 'name': 'tricycle'} ,
+{'id': 248, 'name': 'parking meter'} ,
+{'id': 274, 'name': 'trophy'} ,
+{'id': 324, 'name': 'binoculars'} ,
+{'id': 51, 'name': 'traffic light'} ,
+{'id': 314, 'name': 'donkey'} ,
+{'id': 45, 'name': 'barrel/bucket'} ,
+{'id': 292, 'name': 'pomegranate'} ,
+{'id': 13, 'name': 'handbag'} ,
+{'id': 262, 'name': 'tablet'} ,
+{'id': 68, 'name': 'apple'} ,
+{'id': 226, 'name': 'cabbage'} ,
+{'id': 23, 'name': 'flower'} ,
+{'id': 58, 'name': 'faucet'} ,
+{'id': 206, 'name': 'tong'} ,
+{'id': 291, 'name': 'trombone'} ,
+{'id': 160, 'name': 'carrot'} ,
+{'id': 172, 'name': 'bow tie'} ,
+{'id': 122, 'name': 'tent'} ,
+{'id': 163, 'name': 'cookies'} ,
+{'id': 115, 'name': 'remote'} ,
+{'id': 175, 'name': 'coffee machine'} ,
+{'id': 238, 'name': 'green beans'} ,
+{'id': 233, 'name': 'cello'} ,
+{'id': 28, 'name': 'wine glass'} ,
+{'id': 295, 'name': 'mushroom'} ,
+{'id': 344, 'name': 'scallop'} ,
+{'id': 125, 'name': 'lantern'} ,
+{'id': 123, 'name': 'shampoo/shower gel'} ,
+{'id': 285, 'name': 'meat balls'} ,
+{'id': 266, 'name': 'key'} ,
+{'id': 296, 'name': 'calculator'} ,
+{'id': 168, 'name': 'scissors'} ,
+{'id': 103, 'name': 'cymbal'} ,
+{'id': 6, 'name': 'bottle'} ,
+{'id': 264, 'name': 'nuts'} ,
+{'id': 234, 'name': 'notepaper'} ,
+{'id': 211, 'name': 'mango'} ,
+{'id': 287, 'name': 'toothpaste'} ,
+{'id': 196, 'name': 'chopsticks'} ,
+{'id': 140, 'name': 'baseball bat'} ,
+{'id': 244, 'name': 'hurdle'} ,
+{'id': 195, 'name': 'tennis ball'} ,
+{'id': 144, 'name': 'surveillance camera'} ,
+{'id': 271, 'name': 'volleyball'} ,
+{'id': 94, 'name': 'keyboard'} ,
+{'id': 339, 'name': 'seal'} ,
+{'id': 11, 'name': 'picture/frame'} ,
+{'id': 348, 'name': 'okra'} ,
+{'id': 191, 'name': 'sausage'} ,
+{'id': 166, 'name': 'candy'} ,
+{'id': 62, 'name': 'ring'} ,
+{'id': 311, 'name': 'dolphin'} ,
+{'id': 273, 'name': 'eggplant'} ,
+{'id': 84, 'name': 'drum'} ,
+{'id': 143, 'name': 'surfboard'} ,
+{'id': 288, 'name': 'antelope'} ,
+{'id': 204, 'name': 'clutch'} ,
+{'id': 207, 'name': 'slide'} ,
+{'id': 43, 'name': 'towel/napkin'} ,
+{'id': 352, 'name': 'durian'} ,
+{'id': 276, 'name': 'board eraser'} ,
+{'id': 315, 'name': 'electric drill'} ,
+{'id': 312, 'name': 'sushi'} ,
+{'id': 198, 'name': 'pie'} ,
+{'id': 106, 'name': 'pickup truck'} ,
+{'id': 176, 'name': 'bathtub'} ,
+{'id': 26, 'name': 'vase'} ,
+{'id': 133, 'name': 'elephant'} ,
+{'id': 256, 'name': 'sandwich'} ,
+{'id': 327, 'name': 'noodles'} ,
+{'id': 10, 'name': 'glasses'} ,
+{'id': 109, 'name': 'airplane'} ,
+{'id': 95, 'name': 'tripod'} ,
+{'id': 247, 'name': 'CD'} ,
+{'id': 121, 'name': 'machinery vehicle'} ,
+{'id': 365, 'name': 'flashlight'} ,
+{'id': 53, 'name': 'microphone'} ,
+{'id': 270, 'name': 'pliers'} ,
+{'id': 362, 'name': 'chainsaw'} ,
+{'id': 259, 'name': 'bear'} ,
+{'id': 197, 'name': 'electronic stove and gas stove'} ,
+{'id': 89, 'name': 'pot/pan'} ,
+{'id': 220, 'name': 'tape'} ,
+{'id': 338, 'name': 'lighter'} ,
+{'id': 177, 'name': 'snowboard'} ,
+{'id': 214, 'name': 'violin'} ,
+{'id': 217, 'name': 'chicken'} ,
+{'id': 2, 'name': 'sneakers'} ,
+{'id': 161, 'name': 'washing machine'} ,
+{'id': 131, 'name': 'kite'} ,
+{'id': 354, 'name': 'rabbit'} ,
+{'id': 86, 'name': 'bus'} ,
+{'id': 275, 'name': 'dates'} ,
+{'id': 282, 'name': 'camel'} ,
+{'id': 88, 'name': 'nightstand'} ,
+{'id': 179, 'name': 'grapes'} ,
+{'id': 229, 'name': 'pine apple'} ,
+{'id': 56, 'name': 'necklace'} ,
+{'id': 18, 'name': 'leather shoes'} ,
+{'id': 358, 'name': 'hoverboard'} ,
+{'id': 345, 'name': 'pencil case'} ,
+{'id': 359, 'name': 'pasta'} ,
+{'id': 157, 'name': 'radiator'} ,
+{'id': 201, 'name': 'hamburger'} ,
+{'id': 268, 'name': 'globe'} ,
+{'id': 332, 'name': 'barbell'} ,
+{'id': 329, 'name': 'mop'} ,
+{'id': 252, 'name': 'horn'} ,
+{'id': 350, 'name': 'eagle'} ,
+{'id': 169, 'name': 'folder'} ,
+{'id': 137, 'name': 'toilet'} ,
+{'id': 5, 'name': 'lamp'} ,
+{'id': 27, 'name': 'bench'} ,
+{'id': 249, 'name': 'swan'} ,
+{'id': 76, 'name': 'knife'} ,
+{'id': 341, 'name': 'comb'} ,
+{'id': 64, 'name': 'watch'} ,
+{'id': 105, 'name': 'telephone'} ,
+{'id': 3, 'name': 'chair'} ,
+{'id': 33, 'name': 'boat'} ,
+{'id': 107, 'name': 'orange'} ,
+{'id': 60, 'name': 'bread'} ,
+{'id': 147, 'name': 'cat'} ,
+{'id': 135, 'name': 'gas stove'} ,
+{'id': 307, 'name': 'papaya'} ,
+{'id': 227, 'name': 'router/modem'} ,
+{'id': 357, 'name': 'asparagus'} ,
+{'id': 73, 'name': 'motorcycle'} ,
+{'id': 77, 'name': 'traffic sign'} ,
+{'id': 67, 'name': 'fish'} ,
+{'id': 326, 'name': 'radish'} ,
+{'id': 213, 'name': 'egg'} ,
+{'id': 203, 'name': 'cucumber'} ,
+{'id': 17, 'name': 'helmet'} ,
+{'id': 110, 'name': 'luggage'} ,
+{'id': 80, 'name': 'truck'} ,
+{'id': 199, 'name': 'frisbee'} ,
+{'id': 232, 'name': 'peach'} ,
+{'id': 1, 'name': 'person'} ,
+{'id': 29, 'name': 'boots'} ,
+{'id': 310, 'name': 'chips'} ,
+{'id': 142, 'name': 'skateboard'} ,
+{'id': 44, 'name': 'slippers'} ,
+{'id': 4, 'name': 'hat'} ,
+{'id': 178, 'name': 'suitcase'} ,
+{'id': 24, 'name': 'tv'} ,
+{'id': 119, 'name': 'train'} ,
+{'id': 82, 'name': 'power outlet'} ,
+{'id': 245, 'name': 'swing'} ,
+{'id': 15, 'name': 'book'} ,
+{'id': 294, 'name': 'jellyfish'} ,
+{'id': 192, 'name': 'fire extinguisher'} ,
+{'id': 212, 'name': 'deer'} ,
+{'id': 181, 'name': 'pear'} ,
+{'id': 347, 'name': 'table tennis paddle'} ,
+{'id': 113, 'name': 'trolley'} ,
+{'id': 91, 'name': 'guitar'} ,
+{'id': 202, 'name': 'golf club'} ,
+{'id': 221, 'name': 'wheelchair'} ,
+{'id': 254, 'name': 'saxophone'} ,
+{'id': 117, 'name': 'paper towel'} ,
+{'id': 303, 'name': 'race car'} ,
+{'id': 240, 'name': 'carriage'} ,
+{'id': 246, 'name': 'radio'} ,
+{'id': 318, 'name': 'parrot'} ,
+{'id': 251, 'name': 'french fries'} ,
+{'id': 98, 'name': 'dog'} ,
+{'id': 112, 'name': 'soccer'} ,
+{'id': 355, 'name': 'french horn'} ,
+{'id': 79, 'name': 'paddle'} ,
+{'id': 283, 'name': 'lettuce'} ,
+{'id': 9, 'name': 'car'} ,
+{'id': 258, 'name': 'kiwi fruit'} ,
+{'id': 325, 'name': 'llama'} ,
+{'id': 187, 'name': 'billiards'} ,
+{'id': 210, 'name': 'facial cleanser'} ,
+{'id': 81, 'name': 'cow'} ,
+{'id': 331, 'name': 'microscope'} ,
+{'id': 148, 'name': 'lemon'} ,
+{'id': 302, 'name': 'pomelo'} ,
+{'id': 85, 'name': 'fork'} ,
+{'id': 154, 'name': 'pumpkin'} ,
+{'id': 289, 'name': 'shrimp'} ,
+{'id': 71, 'name': 'teddy bear'} ,
+{'id': 184, 'name': 'potato'} ,
+{'id': 102, 'name': 'air conditioner'} ,
+{'id': 208, 'name': 'hot dog'} ,
+{'id': 222, 'name': 'plum'} ,
+{'id': 316, 'name': 'spring rolls'} ,
+{'id': 230, 'name': 'crane'} ,
+{'id': 149, 'name': 'liquid soap'} ,
+{'id': 55, 'name': 'canned'} ,
+{'id': 35, 'name': 'speaker'} ,
+{'id': 108, 'name': 'banana'} ,
+{'id': 297, 'name': 'treadmill'} ,
+{'id': 99, 'name': 'spoon'} ,
+{'id': 104, 'name': 'mouse'} ,
+{'id': 182, 'name': 'american football'} ,
+{'id': 299, 'name': 'egg tart'} ,
+{'id': 127, 'name': 'cleaning products'} ,
+{'id': 313, 'name': 'urinal'} ,
+{'id': 286, 'name': 'medal'} ,
+{'id': 239, 'name': 'brush'} ,
+{'id': 96, 'name': 'hockey'} ,
+{'id': 279, 'name': 'dumbbell'} ,
+{'id': 32, 'name': 'umbrella'} ,
+{'id': 272, 'name': 'hammer'} ,
+{'id': 16, 'name': 'plate'} ,
+{'id': 21, 'name': 'potted plant'} ,
+{'id': 242, 'name': 'earphone'} ,
+{'id': 70, 'name': 'candle'} ,
+{'id': 185, 'name': 'paint brush'} ,
+{'id': 48, 'name': 'toy'} ,
+{'id': 130, 'name': 'pizza'} ,
+{'id': 255, 'name': 'trumpet'} ,
+{'id': 361, 'name': 'hotair balloon'} ,
+{'id': 188, 'name': 'fire hydrant'} ,
+{'id': 50, 'name': 'bed'} ,
+{'id': 253, 'name': 'avocado'} ,
+{'id': 293, 'name': 'coconut'} ,
+{'id': 257, 'name': 'cue'} ,
+{'id': 280, 'name': 'hamimelon'} ,
+{'id': 66, 'name': 'horse'} ,
+{'id': 173, 'name': 'pigeon'} ,
+{'id': 190, 'name': 'projector'} ,
+{'id': 69, 'name': 'camera'} ,
+{'id': 30, 'name': 'bowl'} ,
+{'id': 269, 'name': 'broom'} ,
+{'id': 343, 'name': 'pitaya'} ,
+{'id': 305, 'name': 'tuba'} ,
+{'id': 309, 'name': 'green onion'} ,
+{'id': 363, 'name': 'lobster'} ,
+{'id': 225, 'name': 'watermelon'} ,
+{'id': 47, 'name': 'suv'} ,
+{'id': 31, 'name': 'dining table'} ,
+{'id': 54, 'name': 'sandals'} ,
+{'id': 351, 'name': 'monkey'} ,
+{'id': 218, 'name': 'onion'} ,
+{'id': 36, 'name': 'trash bin/can'} ,
+{'id': 20, 'name': 'glove'} ,
+{'id': 277, 'name': 'rice'} ,
+{'id': 152, 'name': 'sports car'} ,
+{'id': 360, 'name': 'target'} ,
+{'id': 205, 'name': 'blender'} ,
+{'id': 19, 'name': 'pillow'} ,
+{'id': 72, 'name': 'cake'} ,
+{'id': 93, 'name': 'tea pot'} ,
+{'id': 353, 'name': 'game board'} ,
+{'id': 38, 'name': 'backpack'} ,
+{'id': 356, 'name': 'ambulance'} ,
+{'id': 146, 'name': 'life saver'} ,
+{'id': 189, 'name': 'goose'} ,
+{'id': 278, 'name': 'tape measure/ruler'} ,
+{'id': 92, 'name': 'traffic cone'} ,
+{'id': 134, 'name': 'toiletries'} ,
+{'id': 114, 'name': 'oven'} ,
+{'id': 317, 'name': 'tortoise/turtle'} ,
+{'id': 265, 'name': 'corn'} ,
+{'id': 126, 'name': 'donut'} ,
+{'id': 57, 'name': 'mirror'} ,
+{'id': 7, 'name': 'cabinet/shelf'} ,
+{'id': 263, 'name': 'green vegetables'} ,
+{'id': 159, 'name': 'tissue '} ,
+{'id': 321, 'name': 'shark'} ,
+{'id': 301, 'name': 'pig'} ,
+{'id': 41, 'name': 'carpet'} ,
+{'id': 304, 'name': 'rice cooker'} ,
+{'id': 323, 'name': 'poker card'} ,
+]
+
+def _get_builtin_metadata():
+    id_to_name = {x['id']: x['name'] for x in categories}
+    thing_dataset_id_to_contiguous_id = {i + 1: i for i in range(365)}
+    thing_classes = [id_to_name[k] for k in sorted(id_to_name)]
+    return {
+        "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id,
+        "thing_classes": thing_classes}
+
+_PREDEFINED_SPLITS_OBJECTS365 = {
+    "objects365_train": ("objects365/train", "objects365/objects365_train.json"),
+    "objects365_val": ("objects365/val", "objects365/objects365_val.json"),
+}
+
+for key, (image_root, json_file) in _PREDEFINED_SPLITS_OBJECTS365.items():
+    register_coco_instances(
+        key,
+        _get_builtin_metadata(),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )

regionspot/data/openimages.py

@@ -0,0 +1,34 @@
+from detectron2.data.datasets.register_coco import register_coco_instances
+import os
+from .openimages_categories import categories
+
+def _get_builtin_metadata(categories):
+    id_to_name = {x['id']: x['name'] for x in categories}
+    thing_dataset_id_to_contiguous_id = {i + 1: i for i in range(len(categories))}
+    thing_classes = [id_to_name[k] for k in sorted(id_to_name)]
+
+    return {
+        "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id,
+        "thing_classes": thing_classes}
+
+def _get_builtin_metadata():
+    id_to_name = {x['id']: x['name'] for x in categories}
+    thing_dataset_id_to_contiguous_id = {i + 1: i for i in range(len(categories))}
+    thing_classes = [id_to_name[k] for k in sorted(id_to_name)]
+    return {
+        "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id,
+        "thing_classes": thing_classes}
+
+
+_PREDEFINED_SPLITS_OPENIMAGES = {
+    "openimages_train": ("openimages/detection/", "re_openimages_v6_train_bbox_splitdir_int_ids.json"),
+    "openimages_val": ("openimages/detection/", "re_openimages_v6_train_bbox_splitdir_int_ids.json"),
+}
+
+for key, (image_root, json_file) in _PREDEFINED_SPLITS_OPENIMAGES.items():
+    register_coco_instances(
+        key,
+        _get_builtin_metadata(),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )

regionspot/data/openimages_categories.py

@@ -0,0 +1 @@
+categories = [{'id': 1, 'name': 'Tortoise', 'freebase_id': '/m/011k07'}, {'id': 2, 'name': 'Container', 'freebase_id': '/m/011q46kg'}, {'id': 3, 'name': 'Magpie', 'freebase_id': '/m/012074'}, {'id': 4, 'name': 'Sea turtle', 'freebase_id': '/m/0120dh'}, {'id': 5, 'name': 'Football', 'freebase_id': '/m/01226z'}, {'id': 6, 'name': 'Ambulance', 'freebase_id': '/m/012n7d'}, {'id': 7, 'name': 'Ladder', 'freebase_id': '/m/012w5l'}, {'id': 8, 'name': 'Toothbrush', 'freebase_id': '/m/012xff'}, {'id': 9, 'name': 'Syringe', 'freebase_id': '/m/012ysf'}, {'id': 10, 'name': 'Sink', 'freebase_id': '/m/0130jx'}, {'id': 11, 'name': 'Toy', 'freebase_id': '/m/0138tl'}, {'id': 12, 'name': 'Organ (Musical Instrument)', 'freebase_id': '/m/013y1f'}, {'id': 13, 'name': 'Cassette deck', 'freebase_id': '/m/01432t'}, {'id': 14, 'name': 'Apple', 'freebase_id': '/m/014j1m'}, {'id': 15, 'name': 'Human eye', 'freebase_id': '/m/014sv8'}, {'id': 16, 'name': 'Cosmetics', 'freebase_id': '/m/014trl'}, {'id': 17, 'name': 'Paddle', 'freebase_id': '/m/014y4n'}, {'id': 18, 'name': 'Snowman', 'freebase_id': '/m/0152hh'}, {'id': 19, 'name': 'Beer', 'freebase_id': '/m/01599'}, {'id': 20, 'name': 'Chopsticks', 'freebase_id': '/m/01_5g'}, {'id': 21, 'name': 'Human beard', 'freebase_id': '/m/015h_t'}, {'id': 22, 'name': 'Bird', 'freebase_id': '/m/015p6'}, {'id': 23, 'name': 'Parking meter', 'freebase_id': '/m/015qbp'}, {'id': 24, 'name': 'Traffic light', 'freebase_id': '/m/015qff'}, {'id': 25, 'name': 'Croissant', 'freebase_id': '/m/015wgc'}, {'id': 26, 'name': 'Cucumber', 'freebase_id': '/m/015x4r'}, {'id': 27, 'name': 'Radish', 'freebase_id': '/m/015x5n'}, {'id': 28, 'name': 'Towel', 'freebase_id': '/m/0162_1'}, {'id': 29, 'name': 'Doll', 'freebase_id': '/m/0167gd'}, {'id': 30, 'name': 'Skull', 'freebase_id': '/m/016m2d'}, {'id': 31, 'name': 'Washing machine', 'freebase_id': '/m/0174k2'}, {'id': 32, 'name': 'Glove', 'freebase_id': '/m/0174n1'}, {'id': 33, 'name': 'Tick', 'freebase_id': '/m/0175cv'}, {'id': 34, 'name': 'Belt', 'freebase_id': '/m/0176mf'}, {'id': 35, 'name': 'Sunglasses', 'freebase_id': '/m/017ftj'}, {'id': 36, 'name': 'Banjo', 'freebase_id': '/m/018j2'}, {'id': 37, 'name': 'Cart', 'freebase_id': '/m/018p4k'}, {'id': 38, 'name': 'Ball', 'freebase_id': '/m/018xm'}, {'id': 39, 'name': 'Backpack', 'freebase_id': '/m/01940j'}, {'id': 40, 'name': 'Bicycle', 'freebase_id': '/m/0199g'}, {'id': 41, 'name': 'Home appliance', 'freebase_id': '/m/019dx1'}, {'id': 42, 'name': 'Centipede', 'freebase_id': '/m/019h78'}, {'id': 43, 'name': 'Boat', 'freebase_id': '/m/019jd'}, {'id': 44, 'name': 'Surfboard', 'freebase_id': '/m/019w40'}, {'id': 45, 'name': 'Boot', 'freebase_id': '/m/01b638'}, {'id': 46, 'name': 'Headphones', 'freebase_id': '/m/01b7fy'}, {'id': 47, 'name': 'Hot dog', 'freebase_id': '/m/01b9xk'}, {'id': 48, 'name': 'Shorts', 'freebase_id': '/m/01bfm9'}, {'id': 49, 'name': 'Fast food', 'freebase_id': '/m/01_bhs'}, {'id': 50, 'name': 'Bus', 'freebase_id': '/m/01bjv'}, {'id': 51, 'name': 'Boy', 'freebase_id': '/m/01bl7v'}, {'id': 52, 'name': 'Screwdriver', 'freebase_id': '/m/01bms0'}, {'id': 53, 'name': 'Bicycle wheel', 'freebase_id': '/m/01bqk0'}, {'id': 54, 'name': 'Barge', 'freebase_id': '/m/01btn'}, {'id': 55, 'name': 'Laptop', 'freebase_id': '/m/01c648'}, {'id': 56, 'name': 'Miniskirt', 'freebase_id': '/m/01cmb2'}, {'id': 57, 'name': 'Drill (Tool)', 'freebase_id': '/m/01d380'}, {'id': 58, 'name': 'Dress', 'freebase_id': '/m/01d40f'}, {'id': 59, 'name': 'Bear', 'freebase_id': '/m/01dws'}, {'id': 60, 'name': 'Waffle', 'freebase_id': '/m/01dwsz'}, {'id': 61, 'name': 'Pancake', 'freebase_id': '/m/01dwwc'}, {'id': 62, 'name': 'Brown bear', 'freebase_id': '/m/01dxs'}, {'id': 63, 'name': 'Woodpecker', 'freebase_id': '/m/01dy8n'}, {'id': 64, 'name': 'Blue jay', 'freebase_id': '/m/01f8m5'}, {'id': 65, 'name': 'Pretzel', 'freebase_id': '/m/01f91_'}, {'id': 66, 'name': 'Bagel', 'freebase_id': '/m/01fb_0'}, {'id': 67, 'name': 'Tower', 'freebase_id': '/m/01fdzj'}, {'id': 68, 'name': 'Teapot', 'freebase_id': '/m/01fh4r'}, {'id': 69, 'name': 'Person', 'freebase_id': '/m/01g317'}, {'id': 70, 'name': 'Bow and arrow', 'freebase_id': '/m/01g3x7'}, {'id': 71, 'name': 'Swimwear', 'freebase_id': '/m/01gkx_'}, {'id': 72, 'name': 'Beehive', 'freebase_id': '/m/01gllr'}, {'id': 73, 'name': 'Brassiere', 'freebase_id': '/m/01gmv2'}, {'id': 74, 'name': 'Bee', 'freebase_id': '/m/01h3n'}, {'id': 75, 'name': 'Bat (Animal)', 'freebase_id': '/m/01h44'}, {'id': 76, 'name': 'Starfish', 'freebase_id': '/m/01h8tj'}, {'id': 77, 'name': 'Popcorn', 'freebase_id': '/m/01hrv5'}, {'id': 78, 'name': 'Burrito', 'freebase_id': '/m/01j3zr'}, {'id': 79, 'name': 'Chainsaw', 'freebase_id': '/m/01j4z9'}, {'id': 80, 'name': 'Balloon', 'freebase_id': '/m/01j51'}, {'id': 81, 'name': 'Wrench', 'freebase_id': '/m/01j5ks'}, {'id': 82, 'name': 'Tent', 'freebase_id': '/m/01j61q'}, {'id': 83, 'name': 'Vehicle registration plate', 'freebase_id': '/m/01jfm_'}, {'id': 84, 'name': 'Lantern', 'freebase_id': '/m/01jfsr'}, {'id': 85, 'name': 'Toaster', 'freebase_id': '/m/01k6s3'}, {'id': 86, 'name': 'Flashlight', 'freebase_id': '/m/01kb5b'}, {'id': 87, 'name': 'Billboard', 'freebase_id': '/m/01knjb'}, {'id': 88, 'name': 'Tiara', 'freebase_id': '/m/01krhy'}, {'id': 89, 'name': 'Limousine', 'freebase_id': '/m/01lcw4'}, {'id': 90, 'name': 'Necklace', 'freebase_id': '/m/01llwg'}, {'id': 91, 'name': 'Carnivore', 'freebase_id': '/m/01lrl'}, {'id': 92, 'name': 'Scissors', 'freebase_id': '/m/01lsmm'}, {'id': 93, 'name': 'Stairs', 'freebase_id': '/m/01lynh'}, {'id': 94, 'name': 'Computer keyboard', 'freebase_id': '/m/01m2v'}, {'id': 95, 'name': 'Printer', 'freebase_id': '/m/01m4t'}, {'id': 96, 'name': 'Traffic sign', 'freebase_id': '/m/01mqdt'}, {'id': 97, 'name': 'Chair', 'freebase_id': '/m/01mzpv'}, {'id': 98, 'name': 'Shirt', 'freebase_id': '/m/01n4qj'}, {'id': 99, 'name': 'Poster', 'freebase_id': '/m/01n5jq'}, {'id': 100, 'name': 'Cheese', 'freebase_id': '/m/01nkt'}, {'id': 101, 'name': 'Sock', 'freebase_id': '/m/01nq26'}, {'id': 102, 'name': 'Fire hydrant', 'freebase_id': '/m/01pns0'}, {'id': 103, 'name': 'Land vehicle', 'freebase_id': '/m/01prls'}, {'id': 104, 'name': 'Earrings', 'freebase_id': '/m/01r546'}, {'id': 105, 'name': 'Tie', 'freebase_id': '/m/01rkbr'}, {'id': 106, 'name': 'Watercraft', 'freebase_id': '/m/01rzcn'}, {'id': 107, 'name': 'Cabinetry', 'freebase_id': '/m/01s105'}, {'id': 108, 'name': 'Suitcase', 'freebase_id': '/m/01s55n'}, {'id': 109, 'name': 'Muffin', 'freebase_id': '/m/01tcjp'}, {'id': 110, 'name': 'Bidet', 'freebase_id': '/m/01vbnl'}, {'id': 111, 'name': 'Snack', 'freebase_id': '/m/01ww8y'}, {'id': 112, 'name': 'Snowmobile', 'freebase_id': '/m/01x3jk'}, {'id': 113, 'name': 'Clock', 'freebase_id': '/m/01x3z'}, {'id': 114, 'name': 'Medical equipment', 'freebase_id': '/m/01xgg_'}, {'id': 115, 'name': 'Cattle', 'freebase_id': '/m/01xq0k1'}, {'id': 116, 'name': 'Cello', 'freebase_id': '/m/01xqw'}, {'id': 117, 'name': 'Jet ski', 'freebase_id': '/m/01xs3r'}, {'id': 118, 'name': 'Camel', 'freebase_id': '/m/01x_v'}, {'id': 119, 'name': 'Coat', 'freebase_id': '/m/01xygc'}, {'id': 120, 'name': 'Suit', 'freebase_id': '/m/01xyhv'}, {'id': 121, 'name': 'Desk', 'freebase_id': '/m/01y9k5'}, {'id': 122, 'name': 'Cat', 'freebase_id': '/m/01yrx'}, {'id': 123, 'name': 'Bronze sculpture', 'freebase_id': '/m/01yx86'}, {'id': 124, 'name': 'Juice', 'freebase_id': '/m/01z1kdw'}, {'id': 125, 'name': 'Gondola', 'freebase_id': '/m/02068x'}, {'id': 126, 'name': 'Beetle', 'freebase_id': '/m/020jm'}, {'id': 127, 'name': 'Cannon', 'freebase_id': '/m/020kz'}, {'id': 128, 'name': 'Computer mouse', 'freebase_id': '/m/020lf'}, {'id': 129, 'name': 'Cookie', 'freebase_id': '/m/021mn'}, {'id': 130, 'name': 'Office building', 'freebase_id': '/m/021sj1'}, {'id': 131, 'name': 'Fountain', 'freebase_id': '/m/0220r2'}, {'id': 132, 'name': 'Coin', 'freebase_id': '/m/0242l'}, {'id': 133, 'name': 'Calculator', 'freebase_id': '/m/024d2'}, {'id': 134, 'name': 'Cocktail', 'freebase_id': '/m/024g6'}, {'id': 135, 'name': 'Computer monitor', 'freebase_id': '/m/02522'}, {'id': 136, 'name': 'Box', 'freebase_id': '/m/025dyy'}, {'id': 137, 'name': 'Stapler', 'freebase_id': '/m/025fsf'}, {'id': 138, 'name': 'Christmas tree', 'freebase_id': '/m/025nd'}, {'id': 139, 'name': 'Cowboy hat', 'freebase_id': '/m/025rp__'}, {'id': 140, 'name': 'Hiking equipment', 'freebase_id': '/m/0268lbt'}, {'id': 141, 'name': 'Studio couch', 'freebase_id': '/m/026qbn5'}, {'id': 142, 'name': 'Drum', 'freebase_id': '/m/026t6'}, {'id': 143, 'name': 'Dessert', 'freebase_id': '/m/0270h'}, {'id': 144, 'name': 'Wine rack', 'freebase_id': '/m/0271qf7'}, {'id': 145, 'name': 'Drink', 'freebase_id': '/m/0271t'}, {'id': 146, 'name': 'Zucchini', 'freebase_id': '/m/027pcv'}, {'id': 147, 'name': 'Ladle', 'freebase_id': '/m/027rl48'}, {'id': 148, 'name': 'Human mouth', 'freebase_id': '/m/0283dt1'}, {'id': 149, 'name': 'Dairy Product', 'freebase_id': '/m/0284d'}, {'id': 150, 'name': 'Dice', 'freebase_id': '/m/029b3'}, {'id': 151, 'name': 'Oven', 'freebase_id': '/m/029bxz'}, {'id': 152, 'name': 'Dinosaur', 'freebase_id': '/m/029tx'}, {'id': 153, 'name': 'Ratchet (Device)', 'freebase_id': '/m/02bm9n'}, {'id': 154, 'name': 'Couch', 'freebase_id': '/m/02crq1'}, {'id': 155, 'name': 'Cricket ball', 'freebase_id': '/m/02ctlc'}, {'id': 156, 'name': 'Winter melon', 'freebase_id': '/m/02cvgx'}, {'id': 157, 'name': 'Spatula', 'freebase_id': '/m/02d1br'}, {'id': 158, 'name': 'Whiteboard', 'freebase_id': '/m/02d9qx'}, {'id': 159, 'name': 'Pencil sharpener', 'freebase_id': '/m/02ddwp'}, {'id': 160, 'name': 'Door', 'freebase_id': '/m/02dgv'}, {'id': 161, 'name': 'Hat', 'freebase_id': '/m/02dl1y'}, {'id': 162, 'name': 'Shower', 'freebase_id': '/m/02f9f_'}, {'id': 163, 'name': 'Eraser', 'freebase_id': '/m/02fh7f'}, {'id': 164, 'name': 'Fedora', 'freebase_id': '/m/02fq_6'}, {'id': 165, 'name': 'Guacamole', 'freebase_id': '/m/02g30s'}, {'id': 166, 'name': 'Dagger', 'freebase_id': '/m/02gzp'}, {'id': 167, 'name': 'Scarf', 'freebase_id': '/m/02h19r'}, {'id': 168, 'name': 'Dolphin', 'freebase_id': '/m/02hj4'}, {'id': 169, 'name': 'Sombrero', 'freebase_id': '/m/02jfl0'}, {'id': 170, 'name': 'Tin can', 'freebase_id': '/m/02jnhm'}, {'id': 171, 'name': 'Mug', 'freebase_id': '/m/02jvh9'}, {'id': 172, 'name': 'Tap', 'freebase_id': '/m/02jz0l'}, {'id': 173, 'name': 'Harbor seal', 'freebase_id': '/m/02l8p9'}, {'id': 174, 'name': 'Stretcher', 'freebase_id': '/m/02lbcq'}, {'id': 175, 'name': 'Can opener', 'freebase_id': '/m/02mqfb'}, {'id': 176, 'name': 'Goggles', 'freebase_id': '/m/02_n6y'}, {'id': 177, 'name': 'Human body', 'freebase_id': '/m/02p0tk3'}, {'id': 178, 'name': 'Roller skates', 'freebase_id': '/m/02p3w7d'}, {'id': 179, 'name': 'Coffee cup', 'freebase_id': '/m/02p5f1q'}, {'id': 180, 'name': 'Cutting board', 'freebase_id': '/m/02pdsw'}, {'id': 181, 'name': 'Blender', 'freebase_id': '/m/02pjr4'}, {'id': 182, 'name': 'Plumbing fixture', 'freebase_id': '/m/02pkr5'}, {'id': 183, 'name': 'Stop sign', 'freebase_id': '/m/02pv19'}, {'id': 184, 'name': 'Office supplies', 'freebase_id': '/m/02rdsp'}, {'id': 185, 'name': 'Volleyball (Ball)', 'freebase_id': '/m/02rgn06'}, {'id': 186, 'name': 'Vase', 'freebase_id': '/m/02s195'}, {'id': 187, 'name': 'Slow cooker', 'freebase_id': '/m/02tsc9'}, {'id': 188, 'name': 'Wardrobe', 'freebase_id': '/m/02vkqh8'}, {'id': 189, 'name': 'Coffee', 'freebase_id': '/m/02vqfm'}, {'id': 190, 'name': 'Whisk', 'freebase_id': '/m/02vwcm'}, {'id': 191, 'name': 'Paper towel', 'freebase_id': '/m/02w3r3'}, {'id': 192, 'name': 'Personal care', 'freebase_id': '/m/02w3_ws'}, {'id': 193, 'name': 'Food', 'freebase_id': '/m/02wbm'}, {'id': 194, 'name': 'Sun hat', 'freebase_id': '/m/02wbtzl'}, {'id': 195, 'name': 'Tree house', 'freebase_id': '/m/02wg_p'}, {'id': 196, 'name': 'Flying disc', 'freebase_id': '/m/02wmf'}, {'id': 197, 'name': 'Skirt', 'freebase_id': '/m/02wv6h6'}, {'id': 198, 'name': 'Gas stove', 'freebase_id': '/m/02wv84t'}, {'id': 199, 'name': 'Salt and pepper shakers', 'freebase_id': '/m/02x8cch'}, {'id': 200, 'name': 'Mechanical fan', 'freebase_id': '/m/02x984l'}, {'id': 201, 'name': 'Face powder', 'freebase_id': '/m/02xb7qb'}, {'id': 202, 'name': 'Fax', 'freebase_id': '/m/02xqq'}, {'id': 203, 'name': 'Fruit', 'freebase_id': '/m/02xwb'}, {'id': 204, 'name': 'French fries', 'freebase_id': '/m/02y6n'}, {'id': 205, 'name': 'Nightstand', 'freebase_id': '/m/02z51p'}, {'id': 206, 'name': 'Barrel', 'freebase_id': '/m/02zn6n'}, {'id': 207, 'name': 'Kite', 'freebase_id': '/m/02zt3'}, {'id': 208, 'name': 'Tart', 'freebase_id': '/m/02zvsm'}, {'id': 209, 'name': 'Treadmill', 'freebase_id': '/m/030610'}, {'id': 210, 'name': 'Fox', 'freebase_id': '/m/0306r'}, {'id': 211, 'name': 'Flag', 'freebase_id': '/m/03120'}, {'id': 212, 'name': 'French horn', 'freebase_id': '/m/0319l'}, {'id': 213, 'name': 'Window blind', 'freebase_id': '/m/031b6r'}, {'id': 214, 'name': 'Human foot', 'freebase_id': '/m/031n1'}, {'id': 215, 'name': 'Golf cart', 'freebase_id': '/m/0323sq'}, {'id': 216, 'name': 'Jacket', 'freebase_id': '/m/032b3c'}, {'id': 217, 'name': 'Egg (Food)', 'freebase_id': '/m/033cnk'}, {'id': 218, 'name': 'Street light', 'freebase_id': '/m/033rq4'}, {'id': 219, 'name': 'Guitar', 'freebase_id': '/m/0342h'}, {'id': 220, 'name': 'Pillow', 'freebase_id': '/m/034c16'}, {'id': 221, 'name': 'Human leg', 'freebase_id': '/m/035r7c'}, {'id': 222, 'name': 'Isopod', 'freebase_id': '/m/035vxb'}, {'id': 223, 'name': 'Grape', 'freebase_id': '/m/0388q'}, {'id': 224, 'name': 'Human ear', 'freebase_id': '/m/039xj_'}, {'id': 225, 'name': 'Power plugs and sockets', 'freebase_id': '/m/03bbps'}, {'id': 226, 'name': 'Panda', 'freebase_id': '/m/03bj1'}, {'id': 227, 'name': 'Giraffe', 'freebase_id': '/m/03bk1'}, {'id': 228, 'name': 'Woman', 'freebase_id': '/m/03bt1vf'}, {'id': 229, 'name': 'Door handle', 'freebase_id': '/m/03c7gz'}, {'id': 230, 'name': 'Rhinoceros', 'freebase_id': '/m/03d443'}, {'id': 231, 'name': 'Bathtub', 'freebase_id': '/m/03dnzn'}, {'id': 232, 'name': 'Goldfish', 'freebase_id': '/m/03fj2'}, {'id': 233, 'name': 'Houseplant', 'freebase_id': '/m/03fp41'}, {'id': 234, 'name': 'Goat', 'freebase_id': '/m/03fwl'}, {'id': 235, 'name': 'Baseball bat', 'freebase_id': '/m/03g8mr'}, {'id': 236, 'name': 'Baseball glove', 'freebase_id': '/m/03grzl'}, {'id': 237, 'name': 'Mixing bowl', 'freebase_id': '/m/03hj559'}, {'id': 238, 'name': 'Marine invertebrates', 'freebase_id': '/m/03hl4l9'}, {'id': 239, 'name': 'Kitchen utensil', 'freebase_id': '/m/03hlz0c'}, {'id': 240, 'name': 'Light switch', 'freebase_id': '/m/03jbxj'}, {'id': 241, 'name': 'House', 'freebase_id': '/m/03jm5'}, {'id': 242, 'name': 'Horse', 'freebase_id': '/m/03k3r'}, {'id': 243, 'name': 'Stationary bicycle', 'freebase_id': '/m/03kt2w'}, {'id': 244, 'name': 'Hammer', 'freebase_id': '/m/03l9g'}, {'id': 245, 'name': 'Ceiling fan', 'freebase_id': '/m/03ldnb'}, {'id': 246, 'name': 'Sofa bed', 'freebase_id': '/m/03m3pdh'}, {'id': 247, 'name': 'Adhesive tape', 'freebase_id': '/m/03m3vtv'}, {'id': 248, 'name': 'Harp', 'freebase_id': '/m/03m5k'}, {'id': 249, 'name': 'Sandal', 'freebase_id': '/m/03nfch'}, {'id': 250, 'name': 'Bicycle helmet', 'freebase_id': '/m/03p3bw'}, {'id': 251, 'name': 'Saucer', 'freebase_id': '/m/03q5c7'}, {'id': 252, 'name': 'Harpsichord', 'freebase_id': '/m/03q5t'}, {'id': 253, 'name': 'Human hair', 'freebase_id': '/m/03q69'}, {'id': 254, 'name': 'Heater', 'freebase_id': '/m/03qhv5'}, {'id': 255, 'name': 'Harmonica', 'freebase_id': '/m/03qjg'}, {'id': 256, 'name': 'Hamster', 'freebase_id': '/m/03qrc'}, {'id': 257, 'name': 'Curtain', 'freebase_id': '/m/03rszm'}, {'id': 258, 'name': 'Bed', 'freebase_id': '/m/03ssj5'}, {'id': 259, 'name': 'Kettle', 'freebase_id': '/m/03s_tn'}, {'id': 260, 'name': 'Fireplace', 'freebase_id': '/m/03tw93'}, {'id': 261, 'name': 'Scale', 'freebase_id': '/m/03txqz'}, {'id': 262, 'name': 'Drinking straw', 'freebase_id': '/m/03v5tg'}, {'id': 263, 'name': 'Insect', 'freebase_id': '/m/03vt0'}, {'id': 264, 'name': 'Hair dryer', 'freebase_id': '/m/03wvsk'}, {'id': 265, 'name': 'Kitchenware', 'freebase_id': '/m/03_wxk'}, {'id': 266, 'name': 'Indoor rower', 'freebase_id': '/m/03wym'}, {'id': 267, 'name': 'Invertebrate', 'freebase_id': '/m/03xxp'}, {'id': 268, 'name': 'Food processor', 'freebase_id': '/m/03y6mg'}, {'id': 269, 'name': 'Bookcase', 'freebase_id': '/m/03__z0'}, {'id': 270, 'name': 'Refrigerator', 'freebase_id': '/m/040b_t'}, {'id': 271, 'name': 'Wood-burning stove', 'freebase_id': '/m/04169hn'}, {'id': 272, 'name': 'Punching bag', 'freebase_id': '/m/0420v5'}, {'id': 273, 'name': 'Common fig', 'freebase_id': '/m/043nyj'}, {'id': 274, 'name': 'Cocktail shaker', 'freebase_id': '/m/0440zs'}, {'id': 275, 'name': 'Jaguar (Animal)', 'freebase_id': '/m/0449p'}, {'id': 276, 'name': 'Golf ball', 'freebase_id': '/m/044r5d'}, {'id': 277, 'name': 'Fashion accessory', 'freebase_id': '/m/0463sg'}, {'id': 278, 'name': 'Alarm clock', 'freebase_id': '/m/046dlr'}, {'id': 279, 'name': 'Filing cabinet', 'freebase_id': '/m/047j0r'}, {'id': 280, 'name': 'Artichoke', 'freebase_id': '/m/047v4b'}, {'id': 281, 'name': 'Table', 'freebase_id': '/m/04bcr3'}, {'id': 282, 'name': 'Tableware', 'freebase_id': '/m/04brg2'}, {'id': 283, 'name': 'Kangaroo', 'freebase_id': '/m/04c0y'}, {'id': 284, 'name': 'Koala', 'freebase_id': '/m/04cp_'}, {'id': 285, 'name': 'Knife', 'freebase_id': '/m/04ctx'}, {'id': 286, 'name': 'Bottle', 'freebase_id': '/m/04dr76w'}, {'id': 287, 'name': 'Bottle opener', 'freebase_id': '/m/04f5ws'}, {'id': 288, 'name': 'Lynx', 'freebase_id': '/m/04g2r'}, {'id': 289, 'name': 'Lavender (Plant)', 'freebase_id': '/m/04gth'}, {'id': 290, 'name': 'Lighthouse', 'freebase_id': '/m/04h7h'}, {'id': 291, 'name': 'Dumbbell', 'freebase_id': '/m/04h8sr'}, {'id': 292, 'name': 'Human head', 'freebase_id': '/m/04hgtk'}, {'id': 293, 'name': 'Bowl', 'freebase_id': '/m/04kkgm'}, {'id': 294, 'name': 'Humidifier', 'freebase_id': '/m/04lvq_'}, {'id': 295, 'name': 'Porch', 'freebase_id': '/m/04m6gz'}, {'id': 296, 'name': 'Lizard', 'freebase_id': '/m/04m9y'}, {'id': 297, 'name': 'Billiard table', 'freebase_id': '/m/04p0qw'}, {'id': 298, 'name': 'Mammal', 'freebase_id': '/m/04rky'}, {'id': 299, 'name': 'Mouse', 'freebase_id': '/m/04rmv'}, {'id': 300, 'name': 'Motorcycle', 'freebase_id': '/m/04_sv'}, {'id': 301, 'name': 'Musical instrument', 'freebase_id': '/m/04szw'}, {'id': 302, 'name': 'Swim cap', 'freebase_id': '/m/04tn4x'}, {'id': 303, 'name': 'Frying pan', 'freebase_id': '/m/04v6l4'}, {'id': 304, 'name': 'Snowplow', 'freebase_id': '/m/04vv5k'}, {'id': 305, 'name': 'Bathroom cabinet', 'freebase_id': '/m/04y4h8h'}, {'id': 306, 'name': 'Missile', 'freebase_id': '/m/04ylt'}, {'id': 307, 'name': 'Bust', 'freebase_id': '/m/04yqq2'}, {'id': 308, 'name': 'Man', 'freebase_id': '/m/04yx4'}, {'id': 309, 'name': 'Waffle iron', 'freebase_id': '/m/04z4wx'}, {'id': 310, 'name': 'Milk', 'freebase_id': '/m/04zpv'}, {'id': 311, 'name': 'Ring binder', 'freebase_id': '/m/04zwwv'}, {'id': 312, 'name': 'Plate', 'freebase_id': '/m/050gv4'}, {'id': 313, 'name': 'Mobile phone', 'freebase_id': '/m/050k8'}, {'id': 314, 'name': 'Baked goods', 'freebase_id': '/m/052lwg6'}, {'id': 315, 'name': 'Mushroom', 'freebase_id': '/m/052sf'}, {'id': 316, 'name': 'Crutch', 'freebase_id': '/m/05441v'}, {'id': 317, 'name': 'Pitcher (Container)', 'freebase_id': '/m/054fyh'}, {'id': 318, 'name': 'Mirror', 'freebase_id': '/m/054_l'}, {'id': 319, 'name': 'Personal flotation device', 'freebase_id': '/m/054xkw'}, {'id': 320, 'name': 'Table tennis racket', 'freebase_id': '/m/05_5p_0'}, {'id': 321, 'name': 'Pencil case', 'freebase_id': '/m/05676x'}, {'id': 322, 'name': 'Musical keyboard', 'freebase_id': '/m/057cc'}, {'id': 323, 'name': 'Scoreboard', 'freebase_id': '/m/057p5t'}, {'id': 324, 'name': 'Briefcase', 'freebase_id': '/m/0584n8'}, {'id': 325, 'name': 'Kitchen knife', 'freebase_id': '/m/058qzx'}, {'id': 326, 'name': 'Nail (Construction)', 'freebase_id': '/m/05bm6'}, {'id': 327, 'name': 'Tennis ball', 'freebase_id': '/m/05ctyq'}, {'id': 328, 'name': 'Plastic bag', 'freebase_id': '/m/05gqfk'}, {'id': 329, 'name': 'Oboe', 'freebase_id': '/m/05kms'}, {'id': 330, 'name': 'Chest of drawers', 'freebase_id': '/m/05kyg_'}, {'id': 331, 'name': 'Ostrich', 'freebase_id': '/m/05n4y'}, {'id': 332, 'name': 'Piano', 'freebase_id': '/m/05r5c'}, {'id': 333, 'name': 'Girl', 'freebase_id': '/m/05r655'}, {'id': 334, 'name': 'Plant', 'freebase_id': '/m/05s2s'}, {'id': 335, 'name': 'Potato', 'freebase_id': '/m/05vtc'}, {'id': 336, 'name': 'Hair spray', 'freebase_id': '/m/05w9t9'}, {'id': 337, 'name': 'Sports equipment', 'freebase_id': '/m/05y5lj'}, {'id': 338, 'name': 'Pasta', 'freebase_id': '/m/05z55'}, {'id': 339, 'name': 'Penguin', 'freebase_id': '/m/05z6w'}, {'id': 340, 'name': 'Pumpkin', 'freebase_id': '/m/05zsy'}, {'id': 341, 'name': 'Pear', 'freebase_id': '/m/061_f'}, {'id': 342, 'name': 'Infant bed', 'freebase_id': '/m/061hd_'}, {'id': 343, 'name': 'Polar bear', 'freebase_id': '/m/0633h'}, {'id': 344, 'name': 'Mixer', 'freebase_id': '/m/063rgb'}, {'id': 345, 'name': 'Cupboard', 'freebase_id': '/m/0642b4'}, {'id': 346, 'name': 'Jacuzzi', 'freebase_id': '/m/065h6l'}, {'id': 347, 'name': 'Pizza', 'freebase_id': '/m/0663v'}, {'id': 348, 'name': 'Digital clock', 'freebase_id': '/m/06_72j'}, {'id': 349, 'name': 'Pig', 'freebase_id': '/m/068zj'}, {'id': 350, 'name': 'Reptile', 'freebase_id': '/m/06bt6'}, {'id': 351, 'name': 'Rifle', 'freebase_id': '/m/06c54'}, {'id': 352, 'name': 'Lipstick', 'freebase_id': '/m/06c7f7'}, {'id': 353, 'name': 'Skateboard', 'freebase_id': '/m/06_fw'}, {'id': 354, 'name': 'Raven', 'freebase_id': '/m/06j2d'}, {'id': 355, 'name': 'High heels', 'freebase_id': '/m/06k2mb'}, {'id': 356, 'name': 'Red panda', 'freebase_id': '/m/06l9r'}, {'id': 357, 'name': 'Rose', 'freebase_id': '/m/06m11'}, {'id': 358, 'name': 'Rabbit', 'freebase_id': '/m/06mf6'}, {'id': 359, 'name': 'Sculpture', 'freebase_id': '/m/06msq'}, {'id': 360, 'name': 'Saxophone', 'freebase_id': '/m/06ncr'}, {'id': 361, 'name': 'Shotgun', 'freebase_id': '/m/06nrc'}, {'id': 362, 'name': 'Seafood', 'freebase_id': '/m/06nwz'}, {'id': 363, 'name': 'Submarine sandwich', 'freebase_id': '/m/06pcq'}, {'id': 364, 'name': 'Snowboard', 'freebase_id': '/m/06__v'}, {'id': 365, 'name': 'Sword', 'freebase_id': '/m/06y5r'}, {'id': 366, 'name': 'Picture frame', 'freebase_id': '/m/06z37_'}, {'id': 367, 'name': 'Sushi', 'freebase_id': '/m/07030'}, {'id': 368, 'name': 'Loveseat', 'freebase_id': '/m/0703r8'}, {'id': 369, 'name': 'Ski', 'freebase_id': '/m/071p9'}, {'id': 370, 'name': 'Squirrel', 'freebase_id': '/m/071qp'}, {'id': 371, 'name': 'Tripod', 'freebase_id': '/m/073bxn'}, {'id': 372, 'name': 'Stethoscope', 'freebase_id': '/m/073g6'}, {'id': 373, 'name': 'Submarine', 'freebase_id': '/m/074d1'}, {'id': 374, 'name': 'Scorpion', 'freebase_id': '/m/0755b'}, {'id': 375, 'name': 'Segway', 'freebase_id': '/m/076bq'}, {'id': 376, 'name': 'Training bench', 'freebase_id': '/m/076lb9'}, {'id': 377, 'name': 'Snake', 'freebase_id': '/m/078jl'}, {'id': 378, 'name': 'Coffee table', 'freebase_id': '/m/078n6m'}, {'id': 379, 'name': 'Skyscraper', 'freebase_id': '/m/079cl'}, {'id': 380, 'name': 'Sheep', 'freebase_id': '/m/07bgp'}, {'id': 381, 'name': 'Television', 'freebase_id': '/m/07c52'}, {'id': 382, 'name': 'Trombone', 'freebase_id': '/m/07c6l'}, {'id': 383, 'name': 'Tea', 'freebase_id': '/m/07clx'}, {'id': 384, 'name': 'Tank', 'freebase_id': '/m/07cmd'}, {'id': 385, 'name': 'Taco', 'freebase_id': '/m/07crc'}, {'id': 386, 'name': 'Telephone', 'freebase_id': '/m/07cx4'}, {'id': 387, 'name': 'Torch', 'freebase_id': '/m/07dd4'}, {'id': 388, 'name': 'Tiger', 'freebase_id': '/m/07dm6'}, {'id': 389, 'name': 'Strawberry', 'freebase_id': '/m/07fbm7'}, {'id': 390, 'name': 'Trumpet', 'freebase_id': '/m/07gql'}, {'id': 391, 'name': 'Tree', 'freebase_id': '/m/07j7r'}, {'id': 392, 'name': 'Tomato', 'freebase_id': '/m/07j87'}, {'id': 393, 'name': 'Train', 'freebase_id': '/m/07jdr'}, {'id': 394, 'name': 'Tool', 'freebase_id': '/m/07k1x'}, {'id': 395, 'name': 'Picnic basket', 'freebase_id': '/m/07kng9'}, {'id': 396, 'name': 'Cooking spray', 'freebase_id': '/m/07mcwg'}, {'id': 397, 'name': 'Trousers', 'freebase_id': '/m/07mhn'}, {'id': 398, 'name': 'Bowling equipment', 'freebase_id': '/m/07pj7bq'}, {'id': 399, 'name': 'Football helmet', 'freebase_id': '/m/07qxg_'}, {'id': 400, 'name': 'Truck', 'freebase_id': '/m/07r04'}, {'id': 401, 'name': 'Measuring cup', 'freebase_id': '/m/07v9_z'}, {'id': 402, 'name': 'Coffeemaker', 'freebase_id': '/m/07xyvk'}, {'id': 403, 'name': 'Violin', 'freebase_id': '/m/07y_7'}, {'id': 404, 'name': 'Vehicle', 'freebase_id': '/m/07yv9'}, {'id': 405, 'name': 'Handbag', 'freebase_id': '/m/080hkjn'}, {'id': 406, 'name': 'Paper cutter', 'freebase_id': '/m/080n7g'}, {'id': 407, 'name': 'Wine', 'freebase_id': '/m/081qc'}, {'id': 408, 'name': 'Weapon', 'freebase_id': '/m/083kb'}, {'id': 409, 'name': 'Wheel', 'freebase_id': '/m/083wq'}, {'id': 410, 'name': 'Worm', 'freebase_id': '/m/084hf'}, {'id': 411, 'name': 'Wok', 'freebase_id': '/m/084rd'}, {'id': 412, 'name': 'Whale', 'freebase_id': '/m/084zz'}, {'id': 413, 'name': 'Zebra', 'freebase_id': '/m/0898b'}, {'id': 414, 'name': 'Auto part', 'freebase_id': '/m/08dz3q'}, {'id': 415, 'name': 'Jug', 'freebase_id': '/m/08hvt4'}, {'id': 416, 'name': 'Pizza cutter', 'freebase_id': '/m/08ks85'}, {'id': 417, 'name': 'Cream', 'freebase_id': '/m/08p92x'}, {'id': 418, 'name': 'Monkey', 'freebase_id': '/m/08pbxl'}, {'id': 419, 'name': 'Lion', 'freebase_id': '/m/096mb'}, {'id': 420, 'name': 'Bread', 'freebase_id': '/m/09728'}, {'id': 421, 'name': 'Platter', 'freebase_id': '/m/099ssp'}, {'id': 422, 'name': 'Chicken', 'freebase_id': '/m/09b5t'}, {'id': 423, 'name': 'Eagle', 'freebase_id': '/m/09csl'}, {'id': 424, 'name': 'Helicopter', 'freebase_id': '/m/09ct_'}, {'id': 425, 'name': 'Owl', 'freebase_id': '/m/09d5_'}, {'id': 426, 'name': 'Duck', 'freebase_id': '/m/09ddx'}, {'id': 427, 'name': 'Turtle', 'freebase_id': '/m/09dzg'}, {'id': 428, 'name': 'Hippopotamus', 'freebase_id': '/m/09f20'}, {'id': 429, 'name': 'Crocodile', 'freebase_id': '/m/09f_2'}, {'id': 430, 'name': 'Toilet', 'freebase_id': '/m/09g1w'}, {'id': 431, 'name': 'Toilet paper', 'freebase_id': '/m/09gtd'}, {'id': 432, 'name': 'Squid', 'freebase_id': '/m/09gys'}, {'id': 433, 'name': 'Clothing', 'freebase_id': '/m/09j2d'}, {'id': 434, 'name': 'Footwear', 'freebase_id': '/m/09j5n'}, {'id': 435, 'name': 'Lemon', 'freebase_id': '/m/09k_b'}, {'id': 436, 'name': 'Spider', 'freebase_id': '/m/09kmb'}, {'id': 437, 'name': 'Deer', 'freebase_id': '/m/09kx5'}, {'id': 438, 'name': 'Frog', 'freebase_id': '/m/09ld4'}, {'id': 439, 'name': 'Banana', 'freebase_id': '/m/09qck'}, {'id': 440, 'name': 'Rocket', 'freebase_id': '/m/09rvcxw'}, {'id': 441, 'name': 'Wine glass', 'freebase_id': '/m/09tvcd'}, {'id': 442, 'name': 'Countertop', 'freebase_id': '/m/0b3fp9'}, {'id': 443, 'name': 'Tablet computer', 'freebase_id': '/m/0bh9flk'}, {'id': 444, 'name': 'Waste container', 'freebase_id': '/m/0bjyj5'}, {'id': 445, 'name': 'Swimming pool', 'freebase_id': '/m/0b_rs'}, {'id': 446, 'name': 'Dog', 'freebase_id': '/m/0bt9lr'}, {'id': 447, 'name': 'Book', 'freebase_id': '/m/0bt_c3'}, {'id': 448, 'name': 'Elephant', 'freebase_id': '/m/0bwd_0j'}, {'id': 449, 'name': 'Shark', 'freebase_id': '/m/0by6g'}, {'id': 450, 'name': 'Candle', 'freebase_id': '/m/0c06p'}, {'id': 451, 'name': 'Leopard', 'freebase_id': '/m/0c29q'}, {'id': 452, 'name': 'Axe', 'freebase_id': '/m/0c2jj'}, {'id': 453, 'name': 'Hand dryer', 'freebase_id': '/m/0c3m8g'}, {'id': 454, 'name': 'Soap dispenser', 'freebase_id': '/m/0c3mkw'}, {'id': 455, 'name': 'Porcupine', 'freebase_id': '/m/0c568'}, {'id': 456, 'name': 'Flower', 'freebase_id': '/m/0c9ph5'}, {'id': 457, 'name': 'Canary', 'freebase_id': '/m/0ccs93'}, {'id': 458, 'name': 'Cheetah', 'freebase_id': '/m/0cd4d'}, {'id': 459, 'name': 'Palm tree', 'freebase_id': '/m/0cdl1'}, {'id': 460, 'name': 'Hamburger', 'freebase_id': '/m/0cdn1'}, {'id': 461, 'name': 'Maple', 'freebase_id': '/m/0cffdh'}, {'id': 462, 'name': 'Building', 'freebase_id': '/m/0cgh4'}, {'id': 463, 'name': 'Fish', 'freebase_id': '/m/0ch_cf'}, {'id': 464, 'name': 'Lobster', 'freebase_id': '/m/0cjq5'}, {'id': 465, 'name': 'Garden Asparagus', 'freebase_id': '/m/0cjs7'}, {'id': 466, 'name': 'Furniture', 'freebase_id': '/m/0c_jw'}, {'id': 467, 'name': 'Hedgehog', 'freebase_id': '/m/0cl4p'}, {'id': 468, 'name': 'Airplane', 'freebase_id': '/m/0cmf2'}, {'id': 469, 'name': 'Spoon', 'freebase_id': '/m/0cmx8'}, {'id': 470, 'name': 'Otter', 'freebase_id': '/m/0cn6p'}, {'id': 471, 'name': 'Bull', 'freebase_id': '/m/0cnyhnx'}, {'id': 472, 'name': 'Oyster', 'freebase_id': '/m/0_cp5'}, {'id': 473, 'name': 'Horizontal bar', 'freebase_id': '/m/0cqn2'}, {'id': 474, 'name': 'Convenience store', 'freebase_id': '/m/0crjs'}, {'id': 475, 'name': 'Bomb', 'freebase_id': '/m/0ct4f'}, {'id': 476, 'name': 'Bench', 'freebase_id': '/m/0cvnqh'}, {'id': 477, 'name': 'Ice cream', 'freebase_id': '/m/0cxn2'}, {'id': 478, 'name': 'Caterpillar', 'freebase_id': '/m/0cydv'}, {'id': 479, 'name': 'Butterfly', 'freebase_id': '/m/0cyf8'}, {'id': 480, 'name': 'Parachute', 'freebase_id': '/m/0cyfs'}, {'id': 481, 'name': 'Orange', 'freebase_id': '/m/0cyhj_'}, {'id': 482, 'name': 'Antelope', 'freebase_id': '/m/0czz2'}, {'id': 483, 'name': 'Beaker', 'freebase_id': '/m/0d20w4'}, {'id': 484, 'name': 'Moths and butterflies', 'freebase_id': '/m/0d_2m'}, {'id': 485, 'name': 'Window', 'freebase_id': '/m/0d4v4'}, {'id': 486, 'name': 'Closet', 'freebase_id': '/m/0d4w1'}, {'id': 487, 'name': 'Castle', 'freebase_id': '/m/0d5gx'}, {'id': 488, 'name': 'Jellyfish', 'freebase_id': '/m/0d8zb'}, {'id': 489, 'name': 'Goose', 'freebase_id': '/m/0dbvp'}, {'id': 490, 'name': 'Mule', 'freebase_id': '/m/0dbzx'}, {'id': 491, 'name': 'Swan', 'freebase_id': '/m/0dftk'}, {'id': 492, 'name': 'Peach', 'freebase_id': '/m/0dj6p'}, {'id': 493, 'name': 'Coconut', 'freebase_id': '/m/0djtd'}, {'id': 494, 'name': 'Seat belt', 'freebase_id': '/m/0dkzw'}, {'id': 495, 'name': 'Raccoon', 'freebase_id': '/m/0dq75'}, {'id': 496, 'name': 'Chisel', 'freebase_id': '/m/0_dqb'}, {'id': 497, 'name': 'Fork', 'freebase_id': '/m/0dt3t'}, {'id': 498, 'name': 'Lamp', 'freebase_id': '/m/0dtln'}, {'id': 499, 'name': 'Camera', 'freebase_id': '/m/0dv5r'}, {'id': 500, 'name': 'Squash (Plant)', 'freebase_id': '/m/0dv77'}, {'id': 501, 'name': 'Racket', 'freebase_id': '/m/0dv9c'}, {'id': 502, 'name': 'Human face', 'freebase_id': '/m/0dzct'}, {'id': 503, 'name': 'Human arm', 'freebase_id': '/m/0dzf4'}, {'id': 504, 'name': 'Vegetable', 'freebase_id': '/m/0f4s2w'}, {'id': 505, 'name': 'Diaper', 'freebase_id': '/m/0f571'}, {'id': 506, 'name': 'Unicycle', 'freebase_id': '/m/0f6nr'}, {'id': 507, 'name': 'Falcon', 'freebase_id': '/m/0f6wt'}, {'id': 508, 'name': 'Chime', 'freebase_id': '/m/0f8s22'}, {'id': 509, 'name': 'Snail', 'freebase_id': '/m/0f9_l'}, {'id': 510, 'name': 'Shellfish', 'freebase_id': '/m/0fbdv'}, {'id': 511, 'name': 'Cabbage', 'freebase_id': '/m/0fbw6'}, {'id': 512, 'name': 'Carrot', 'freebase_id': '/m/0fj52s'}, {'id': 513, 'name': 'Mango', 'freebase_id': '/m/0fldg'}, {'id': 514, 'name': 'Jeans', 'freebase_id': '/m/0fly7'}, {'id': 515, 'name': 'Flowerpot', 'freebase_id': '/m/0fm3zh'}, {'id': 516, 'name': 'Pineapple', 'freebase_id': '/m/0fp6w'}, {'id': 517, 'name': 'Drawer', 'freebase_id': '/m/0fqfqc'}, {'id': 518, 'name': 'Stool', 'freebase_id': '/m/0fqt361'}, {'id': 519, 'name': 'Envelope', 'freebase_id': '/m/0frqm'}, {'id': 520, 'name': 'Cake', 'freebase_id': '/m/0fszt'}, {'id': 521, 'name': 'Dragonfly', 'freebase_id': '/m/0ft9s'}, {'id': 522, 'name': 'Common sunflower', 'freebase_id': '/m/0ftb8'}, {'id': 523, 'name': 'Microwave oven', 'freebase_id': '/m/0fx9l'}, {'id': 524, 'name': 'Honeycomb', 'freebase_id': '/m/0fz0h'}, {'id': 525, 'name': 'Marine mammal', 'freebase_id': '/m/0gd2v'}, {'id': 526, 'name': 'Sea lion', 'freebase_id': '/m/0gd36'}, {'id': 527, 'name': 'Ladybug', 'freebase_id': '/m/0gj37'}, {'id': 528, 'name': 'Shelf', 'freebase_id': '/m/0gjbg72'}, {'id': 529, 'name': 'Watch', 'freebase_id': '/m/0gjkl'}, {'id': 530, 'name': 'Candy', 'freebase_id': '/m/0gm28'}, {'id': 531, 'name': 'Salad', 'freebase_id': '/m/0grw1'}, {'id': 532, 'name': 'Parrot', 'freebase_id': '/m/0gv1x'}, {'id': 533, 'name': 'Handgun', 'freebase_id': '/m/0gxl3'}, {'id': 534, 'name': 'Sparrow', 'freebase_id': '/m/0h23m'}, {'id': 535, 'name': 'Van', 'freebase_id': '/m/0h2r6'}, {'id': 536, 'name': 'Grinder', 'freebase_id': '/m/0h8jyh6'}, {'id': 537, 'name': 'Spice rack', 'freebase_id': '/m/0h8kx63'}, {'id': 538, 'name': 'Light bulb', 'freebase_id': '/m/0h8l4fh'}, {'id': 539, 'name': 'Corded phone', 'freebase_id': '/m/0h8lkj8'}, {'id': 540, 'name': 'Sports uniform', 'freebase_id': '/m/0h8mhzd'}, {'id': 541, 'name': 'Tennis racket', 'freebase_id': '/m/0h8my_4'}, {'id': 542, 'name': 'Wall clock', 'freebase_id': '/m/0h8mzrc'}, {'id': 543, 'name': 'Serving tray', 'freebase_id': '/m/0h8n27j'}, {'id': 544, 'name': 'Kitchen & dining room table', 'freebase_id': '/m/0h8n5zk'}, {'id': 545, 'name': 'Dog bed', 'freebase_id': '/m/0h8n6f9'}, {'id': 546, 'name': 'Cake stand', 'freebase_id': '/m/0h8n6ft'}, {'id': 547, 'name': 'Cat furniture', 'freebase_id': '/m/0h8nm9j'}, {'id': 548, 'name': 'Bathroom accessory', 'freebase_id': '/m/0h8nr_l'}, {'id': 549, 'name': 'Facial tissue holder', 'freebase_id': '/m/0h8nsvg'}, {'id': 550, 'name': 'Pressure cooker', 'freebase_id': '/m/0h8ntjv'}, {'id': 551, 'name': 'Kitchen appliance', 'freebase_id': '/m/0h99cwc'}, {'id': 552, 'name': 'Tire', 'freebase_id': '/m/0h9mv'}, {'id': 553, 'name': 'Ruler', 'freebase_id': '/m/0hdln'}, {'id': 554, 'name': 'Luggage and bags', 'freebase_id': '/m/0hf58v5'}, {'id': 555, 'name': 'Microphone', 'freebase_id': '/m/0hg7b'}, {'id': 556, 'name': 'Broccoli', 'freebase_id': '/m/0hkxq'}, {'id': 557, 'name': 'Umbrella', 'freebase_id': '/m/0hnnb'}, {'id': 558, 'name': 'Pastry', 'freebase_id': '/m/0hnyx'}, {'id': 559, 'name': 'Grapefruit', 'freebase_id': '/m/0hqkz'}, {'id': 560, 'name': 'Band-aid', 'freebase_id': '/m/0j496'}, {'id': 561, 'name': 'Animal', 'freebase_id': '/m/0jbk'}, {'id': 562, 'name': 'Bell pepper', 'freebase_id': '/m/0jg57'}, {'id': 563, 'name': 'Turkey', 'freebase_id': '/m/0jly1'}, {'id': 564, 'name': 'Lily', 'freebase_id': '/m/0jqgx'}, {'id': 565, 'name': 'Pomegranate', 'freebase_id': '/m/0jwn_'}, {'id': 566, 'name': 'Doughnut', 'freebase_id': '/m/0jy4k'}, {'id': 567, 'name': 'Glasses', 'freebase_id': '/m/0jyfg'}, {'id': 568, 'name': 'Human nose', 'freebase_id': '/m/0k0pj'}, {'id': 569, 'name': 'Pen', 'freebase_id': '/m/0k1tl'}, {'id': 570, 'name': 'Ant', 'freebase_id': '/m/0_k2'}, {'id': 571, 'name': 'Car', 'freebase_id': '/m/0k4j'}, {'id': 572, 'name': 'Aircraft', 'freebase_id': '/m/0k5j'}, {'id': 573, 'name': 'Human hand', 'freebase_id': '/m/0k65p'}, {'id': 574, 'name': 'Skunk', 'freebase_id': '/m/0km7z'}, {'id': 575, 'name': 'Teddy bear', 'freebase_id': '/m/0kmg4'}, {'id': 576, 'name': 'Watermelon', 'freebase_id': '/m/0kpqd'}, {'id': 577, 'name': 'Cantaloupe', 'freebase_id': '/m/0kpt_'}, {'id': 578, 'name': 'Dishwasher', 'freebase_id': '/m/0ky7b'}, {'id': 579, 'name': 'Flute', 'freebase_id': '/m/0l14j_'}, {'id': 580, 'name': 'Balance beam', 'freebase_id': '/m/0l3ms'}, {'id': 581, 'name': 'Sandwich', 'freebase_id': '/m/0l515'}, {'id': 582, 'name': 'Shrimp', 'freebase_id': '/m/0ll1f78'}, {'id': 583, 'name': 'Sewing machine', 'freebase_id': '/m/0llzx'}, {'id': 584, 'name': 'Binoculars', 'freebase_id': '/m/0lt4_'}, {'id': 585, 'name': 'Rays and skates', 'freebase_id': '/m/0m53l'}, {'id': 586, 'name': 'Ipod', 'freebase_id': '/m/0mcx2'}, {'id': 587, 'name': 'Accordion', 'freebase_id': '/m/0mkg'}, {'id': 588, 'name': 'Willow', 'freebase_id': '/m/0mw_6'}, {'id': 589, 'name': 'Crab', 'freebase_id': '/m/0n28_'}, {'id': 590, 'name': 'Crown', 'freebase_id': '/m/0nl46'}, {'id': 591, 'name': 'Seahorse', 'freebase_id': '/m/0nybt'}, {'id': 592, 'name': 'Perfume', 'freebase_id': '/m/0p833'}, {'id': 593, 'name': 'Alpaca', 'freebase_id': '/m/0pcr'}, {'id': 594, 'name': 'Taxi', 'freebase_id': '/m/0pg52'}, {'id': 595, 'name': 'Canoe', 'freebase_id': '/m/0ph39'}, {'id': 596, 'name': 'Remote control', 'freebase_id': '/m/0qjjc'}, {'id': 597, 'name': 'Wheelchair', 'freebase_id': '/m/0qmmr'}, {'id': 598, 'name': 'Rugby ball', 'freebase_id': '/m/0wdt60w'}, {'id': 599, 'name': 'Armadillo', 'freebase_id': '/m/0xfy'}, {'id': 600, 'name': 'Maracas', 'freebase_id': '/m/0xzly'}, {'id': 601, 'name': 'Helmet', 'freebase_id': '/m/0zvk5'}]

regionspot/data/v3det.py

@@ -0,0 +1,34 @@
+from detectron2.data.datasets.register_coco import register_coco_instances
+import os
+
+from .v3det_categories import categories
+def _get_builtin_metadata(categories):
+    id_to_name = {x['id']: x['name'] for x in categories}
+    thing_dataset_id_to_contiguous_id = {i + 1: i for i in range(len(categories))}
+    thing_classes = [id_to_name[k] for k in sorted(id_to_name)]
+
+    return {
+        "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id,
+        "thing_classes": thing_classes}
+
+def _get_builtin_metadata():
+    id_to_name = {x['id']: x['name'] for x in categories}
+    thing_dataset_id_to_contiguous_id = {i + 1: i for i in range(len(categories))}
+    thing_classes = [id_to_name[k] for k in sorted(id_to_name)]
+    return {
+        "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id,
+        "thing_classes": thing_classes}
+
+
+_PREDEFINED_SPLITS_V3DET = {
+    "v3det_train": ("v3det/V3Det/", "v3det/v3det_2023_v1_train.json"),
+    "v3det_val": ("v3det/V3Det/", "v3det/v3det_2023_v1_val.json"),
+}
+
+for key, (image_root, json_file) in _PREDEFINED_SPLITS_V3DET.items():
+    register_coco_instances(
+        key,
+        _get_builtin_metadata(),
+        os.path.join("datasets", json_file) if "://" not in json_file else json_file,
+        os.path.join("datasets", image_root),
+    )

regionspot/detector.py

@@ -0,0 +1,174 @@
+from collections import namedtuple
+from .modeling.regionspot import build_regionspot_model
+import torch.cuda.amp as amp
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+from einops import rearrange
+import json
+from detectron2.modeling import META_ARCH_REGISTRY
+from .util.postprocessing import segmentation_postprocess
+
+from detectron2.structures import Boxes, Instances
+from .util.preprocessing import prepare_prompt_infer, prepare_prompt_train
+
+__all__ = ["RegionSpot"]
+
+
+
+@META_ARCH_REGISTRY.register()
+class RegionSpot(nn.Module):
+    """
+    Implement RegionSpot
+    """
+    def __init__(self, cfg):
+        super().__init__()
+        self.device = torch.device(cfg.MODEL.DEVICE)
+        self.clip_type = cfg.MODEL.CLIP_TYPE
+        self.inference_box_type = cfg.MODEL.BOX_TYPE
+        self.clip_input_size = cfg.MODEL.CLIP_INPUT_SIZE
+        self.clip_target_size = (self.clip_input_size, self.clip_input_size)
+        self.model, _ = build_regionspot_model(clip_type = self.clip_type, is_training=cfg.MODEL.TRAINING, image_size=self.clip_input_size)
+        self.model.to(self.device)
+        if self.inference_box_type != 'GT':
+            path = './datasets/glip_results/nms_results_glip_tiny_model_o365_goldg_cc_sbu_lvis_val.json'
+            with open(path, 'r') as file:
+                self.pred_results = json.load(file)
+        else:
+            self.pred_results = None
+    
+    @torch.no_grad()
+    def foward_inference(self, batched_inputs, do_postprocess=True):
+        with amp.autocast(enabled=True):
+            with torch.no_grad():
+                logits_per_image, pred_mask = self.model.forward_eval(batched_inputs, multimask_output=False)
+        
+        image_sizes = [x["original_size"] for x in batched_inputs]
+        if self.inference_box_type == 'GT':
+            boxes = torch.stack([x["instances"].gt_boxes.tensor for x in batched_inputs], dim=0) #n, n_box, n_token, 256
+            if len(boxes[0]) == 0:
+                boxes=torch.tensor([[[0,0, image_sizes[0][0], image_sizes[0][1]]]])
+        else:
+            boxes = torch.stack([x["pred_boxes"] for x in batched_inputs], dim=0) #n, n_box, n_token, 256
+            scores = torch.stack([x["scores"] for x in batched_inputs], dim=0)
+
+    
+        box_cls = logits_per_image 
+        box_pred = boxes 
+        if self.inference_box_type == 'GT':
+            results = self.inference_gt_box(box_cls, box_pred, pred_mask, image_sizes)
+        else:
+            results = self.inference_pred_box(box_cls, box_pred, scores, pred_mask, image_sizes)
+        if do_postprocess:
+            processed_results = []
+            for results_per_image, input_per_image, image_size in zip(results, batched_inputs, image_sizes):
+                height = input_per_image.get("height", image_size[0])
+                width = input_per_image.get("width", image_size[1])
+                r = segmentation_postprocess(results_per_image, height, width)
+                processed_results.append({"instances": r})
+            return processed_results
+        else:
+            return results
+
+    def foward_train(self, batched_inputs):
+        with amp.autocast(enabled=True):
+            outputs = self.model.forward_train(batched_inputs)
+        loss = {'loss': outputs}
+        return loss
+        
+    def forward(self, batched_inputs, do_postprocess=True):
+        if not self.training:
+            # Prepare Prompt.
+            batched_inputs = prepare_prompt_infer(batched_inputs, pred_results = self.pred_results, target_size=self.clip_target_size)
+
+            results = self.foward_inference(batched_inputs)
+            return results
+
+        if self.training:
+            batched_inputs = prepare_prompt_train(batched_inputs, target_size=self.clip_target_size)
+            loss_dict = self.foward_train(batched_inputs)
+            return loss_dict
+        
+    
+    
+    def inference_gt_box(self, box_cls, box_pred, pred_mask, image_sizes=None):
+
+        device = box_cls.device  # assuming all tensors are on the same device
+        results = []
+
+        for logits, boxes, masks, img_size in zip(box_cls, box_pred, pred_mask, image_sizes):
+            # Calculate probabilities and flatten them
+            probs = F.softmax(logits, dim=-1)
+            probs_flattened = probs.view(-1)
+            
+            # Determine number of top predictions to consider
+            top_num = min(900, len(probs_flattened))
+            
+            # Get top-k values and indices
+            topk_probs, topk_indices = torch.topk(probs_flattened, top_num)
+            
+            # Decode the top-k indices to get corresponding labels and boxes
+            topk_labels = topk_indices % logits.shape[1]
+            topk_boxes_indices = topk_indices // logits.shape[1]
+            
+            # Ensure boxes, masks and topk_boxes_indices are on the same device
+            topk_boxes_indices = topk_boxes_indices.to(device)
+            boxes = boxes.to(device)
+            masks = masks.to(device)
+            
+            # Retrieve predictions using the top-k indices
+            boxes_for_topk = boxes[topk_boxes_indices]
+            masks_for_topk = masks[topk_boxes_indices]
+            scores_for_topk = topk_probs # Modify accordingly if you have another score tensor
+            # Create Instances object for top-k predictions
+            result = Instances(img_size)
+            result.pred_boxes = Boxes(boxes_for_topk)
+            result.scores = scores_for_topk
+            result.pred_classes = topk_labels
+            result.pred_masks = masks_for_topk  # Added masks to the result
+            results.append(result)
+
+        return results
+
+    def inference_pred_box(self, box_cls, box_pred, box_score, masks, image_sizes=None):
+
+        results = []
+
+        for i, (logits, box_pred_i, box_score_i, mask_i, img_size) in enumerate(zip(box_cls, box_pred, box_score, masks, image_sizes)):
+            
+            logits = logits.cuda() 
+            box_pred_i = box_pred_i.cuda() 
+            box_score_i = box_score_i.cuda()
+            
+            # Calculate probabilities and flatten them
+            probs = F.softmax(logits, dim=-1)
+            probs_flattened = probs.view(-1)
+            
+            # Determine number of top predictions to consider
+            top_num = min(900, len(probs_flattened))
+            
+            # Get top-k values and indices
+            topk_probs, topk_indices = torch.topk(probs_flattened, top_num)
+            
+            # Decode the top-k indices to get corresponding labels and boxes
+            topk_labels = topk_indices % logits.shape[1]
+            topk_boxes_indices = topk_indices // logits.shape[1]
+            
+            # Retrieve predictions using the top-k indices
+            boxes = box_pred_i[topk_boxes_indices]
+            masks = mask_i[topk_boxes_indices]
+            scores = box_score_i[topk_boxes_indices] * topk_probs
+            
+            # Construct result for the current image
+            result = Instances(img_size)
+            result.pred_boxes = Boxes(boxes)
+            result.scores = scores
+            result.pred_classes = topk_labels
+            result.pred_masks = masks
+            results.append(result)
+
+        return results
+
+        

regionspot/modeling/clip/__init__.py

@@ -0,0 +1 @@
+from .clip import *