refactor: remove unused modules

lisa_on_cuda/utils/ade20k_classes.json

@@ -1,30 +0,0 @@
-[
-    "wall", "building", "sky", "floor", "tree", "ceiling", "road",
-    "bed", "windowpane", "grass", "cabinet", "sidewalk",
-    "person", "earth", "door", "table", "mountain", "plant",
-    "curtain", "chair", "car", "water", "painting", "sofa",
-    "shelf", "house", "sea", "mirror", "rug", "field", "armchair",
-    "seat", "fence", "desk", "rock", "wardrobe", "lamp",
-    "bathtub", "railing", "cushion", "base", "box", "column",
-    "signboard", "chest of drawers", "counter", "sand", "sink",
-    "skyscraper", "fireplace", "refrigerator", "grandstand",
-    "path", "stairs", "runway", "case", "pool table", "pillow",
-    "screen door", "stairway", "river", "bridge", "bookcase",
-    "blind", "coffee table", "toilet", "flower", "book", "hill",
-    "bench", "countertop", "stove", "palm", "kitchen island",
-    "computer", "swivel chair", "boat", "bar", "arcade machine",
-    "hovel", "bus", "towel", "light", "truck", "tower",
-    "chandelier", "awning", "streetlight", "booth",
-    "television receiver", "airplane", "dirt track", "apparel",
-    "pole", "land", "bannister", "escalator", "ottoman", "bottle",
-    "buffet", "poster", "stage", "van", "ship", "fountain",
-    "conveyer belt", "canopy", "washer", "plaything",
-    "swimming pool", "stool", "barrel", "basket", "waterfall",
-    "tent", "bag", "minibike", "cradle", "oven", "ball", "food",
-    "step", "tank", "trade name", "microwave", "pot", "animal",
-    "bicycle", "lake", "dishwasher", "screen", "blanket",
-    "sculpture", "hood", "sconce", "vase", "traffic light",
-    "tray", "ashcan", "fan", "pier", "crt screen", "plate",
-    "monitor", "bulletin board", "shower", "radiator", "glass",
-    "clock", "flag"
-]

lisa_on_cuda/utils/conversation.py

@@ -1,308 +0,0 @@
-"""
-Conversation prompt templates.
-"""
-
-import dataclasses
-from enum import Enum, auto
-from typing import Any, List
-
-
-class SeparatorStyle(Enum):
-    """Different separator style."""
-
-    ADD_COLON_SINGLE = auto()
-    ADD_COLON_TWO = auto()
-    NO_COLON_SINGLE = auto()
-    BAIZE = auto()
-    DOLLY = auto()
-    RWKV = auto()
-
-
-@dataclasses.dataclass
-class Conversation:
-    """A class that keeps all conversation history."""
-
-    # System prompts
-    system: str
-    # Two roles
-    roles: List[str]
-    # All messages
-    messages: List[List[str]]
-    # Offset of few shot examples
-    offset: int
-    # Separator
-    sep_style: SeparatorStyle
-    sep: str
-    sep2: str = None
-    # Stop criteria (the default one is EOS token)
-    stop_str: str = None
-    # Stops generation if meeting any token in this list
-    stop_token_ids: List[int] = None
-
-    # Used for the state in the gradio servers.
-    # TODO(lmzheng): refactor this
-    conv_id: Any = None
-    skip_next: bool = False
-    model_name: str = None
-
-    def get_prompt(self):
-        if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE:
-            ret = self.system + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ": " + message + self.sep
-                else:
-                    ret += role + ":"
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_COLON_TWO:
-            seps = [self.sep, self.sep2]
-            ret = self.system + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ": " + message + seps[i % 2]
-                else:
-                    ret += role + ":"
-            return ret
-        elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE:
-            ret = self.system
-            for role, message in self.messages:
-                if message:
-                    ret += role + message + self.sep
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.BAIZE:
-            ret = self.system + "\n"
-            for role, message in self.messages:
-                if message:
-                    ret += role + message + "\n"
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.DOLLY:
-            seps = [self.sep, self.sep2]
-            ret = self.system
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ":\n" + message + seps[i % 2]
-                    if i % 2 == 1:
-                        ret += "\n\n"
-                else:
-                    ret += role + ":\n"
-            return ret
-        elif self.sep_style == SeparatorStyle.RWKV:
-            ret = self.system
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += (
-                        role
-                        + ": "
-                        + message.replace("\r\n", "\n").replace("\n\n", "\n")
-                    )
-                    ret += "\n\n"
-                else:
-                    ret += role + ":"
-            return ret
-        else:
-            raise ValueError(f"Invalid style: {self.sep_style}")
-
-    def append_message(self, role, message):
-        self.messages.append([role, message])
-
-    def to_gradio_chatbot(self):
-        ret = []
-        for i, (role, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append([msg, None])
-            else:
-                ret[-1][-1] = msg
-        return ret
-
-    def copy(self):
-        return Conversation(
-            system=self.system,
-            roles=self.roles,
-            messages=[[x, y] for x, y in self.messages],
-            offset=self.offset,
-            sep_style=self.sep_style,
-            sep=self.sep,
-            sep2=self.sep2,
-            stop_str=self.stop_str,
-            stop_token_ids=self.stop_token_ids,
-            conv_id=self.conv_id,
-            model_name=self.model_name,
-        )
-
-    def dict(self):
-        return {
-            "system": self.system,
-            "roles": self.roles,
-            "messages": self.messages,
-            "offset": self.offset,
-            "conv_id": self.conv_id,
-            "model_name": self.model_name,
-        }
-
-
-# A template with one conversation example
-conv_one_shot = Conversation(
-    system="A chat between a curious human and an artificial intelligence assistant. "
-    "The assistant gives helpful, detailed, and polite answers to the human's questions.",
-    roles=("Human", "Assistant"),
-    messages=(
-        (
-            "Human",
-            "What are the key differences between renewable and non-renewable energy sources?",
-        ),
-        (
-            "Assistant",
-            "Renewable energy sources are those that can be replenished naturally in a relatively "
-            "short amount of time, such as solar, wind, hydro, geothermal, and biomass. "
-            "Non-renewable energy sources, on the other hand, are finite and will eventually be "
-            "depleted, such as coal, oil, and natural gas. Here are some key differences between "
-            "renewable and non-renewable energy sources:\n"
-            "1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable "
-            "energy sources are finite and will eventually run out.\n"
-            "2. Environmental impact: Renewable energy sources have a much lower environmental impact "
-            "than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, "
-            "and other negative effects.\n"
-            "3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically "
-            "have lower operational costs than non-renewable sources.\n"
-            "4. Reliability: Renewable energy sources are often more reliable and can be used in more remote "
-            "locations than non-renewable sources.\n"
-            "5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different "
-            "situations and needs, while non-renewable sources are more rigid and inflexible.\n"
-            "6. Sustainability: Renewable energy sources are more sustainable over the long term, while "
-            "non-renewable sources are not, and their depletion can lead to economic and social instability.",
-        ),
-    ),
-    offset=2,
-    sep_style=SeparatorStyle.ADD_COLON_SINGLE,
-    sep="\n### ",
-    stop_str="###",
-)
-
-
-# Vicuna v1.1 template
-conv_vicuna_v1_1 = Conversation(
-    system="A chat between a curious user and an artificial intelligence assistant. "
-    "The assistant gives helpful, detailed, and polite answers to the user's questions.",
-    roles=("USER", "ASSISTANT"),
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.ADD_COLON_TWO,
-    sep=" ",
-    sep2="</s>",
-)
-
-# Koala default template
-conv_koala_v1 = Conversation(
-    system="BEGINNING OF CONVERSATION:",
-    roles=("USER", "GPT"),
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.ADD_COLON_TWO,
-    sep=" ",
-    sep2="</s>",
-)
-
-# Dolly V2 default template
-conv_dolly = Conversation(
-    system="Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n",
-    roles=("### Instruction", "### Response"),
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.DOLLY,
-    sep="\n\n",
-    sep2="### End",
-)
-
-# OpenAssistant Pythia default template
-conv_oasst = Conversation(
-    system="",
-    roles=("<|prompter|>", "<|assistant|>"),
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.NO_COLON_SINGLE,
-    sep="<|endoftext|>",
-)
-
-# StableLM Alpha default template
-conv_stablelm = Conversation(
-    system="""<|SYSTEM|># StableLM Tuned (Alpha version)
-- StableLM is a helpful and harmless open-source AI language model developed by StabilityAI.
-- StableLM is excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.
-- StableLM is more than just an information source, StableLM is also able to write poetry, short stories, and make jokes.
-- StableLM will refuse to participate in anything that could harm a human.
-""",
-    roles=("<|USER|>", "<|ASSISTANT|>"),
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.NO_COLON_SINGLE,
-    sep="",
-    stop_token_ids=[50278, 50279, 50277, 1, 0],
-)
-
-# Baize default template
-conv_baize = Conversation(
-    system="The following is a conversation between a human and an AI assistant named Baize (named after a mythical creature in Chinese folklore). Baize is an open-source AI assistant developed by UCSD and Sun Yat-Sen University. The human and the AI assistant take turns chatting. Human statements start with [|Human|] and AI assistant statements start with [|AI|]. The AI assistant always provides responses in as much detail as possible, and in Markdown format. The AI assistant always declines to engage with topics, questions and instructions related to unethical, controversial, or sensitive issues. Complete the transcript in exactly that format.",
-    roles=("[|Human|]", "[|AI|]"),
-    messages=(
-        ("[|Human|]", "Hello!"),
-        ("[|AI|]", "Hi!"),
-    ),
-    offset=2,
-    sep_style=SeparatorStyle.BAIZE,
-    sep="[|Human|]",
-    stop_str="[|Human|]",
-)
-
-# RWKV-4-Raven default template
-conv_rwkv = Conversation(
-    system="",
-    roles=("Bob", "Alice"),
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.RWKV,
-    sep="",
-    stop_str="\n\n",
-)
-
-conv_templates = {
-    "baize": conv_baize,
-    "conv_one_shot": conv_one_shot,
-    "dolly": conv_dolly,
-    "koala_v1": conv_koala_v1,
-    "oasst": conv_oasst,
-    "stablelm": conv_stablelm,
-    "vicuna_v1.1": conv_vicuna_v1_1,
-    "rwkv": conv_rwkv,
-}
-
-
-def get_default_conv_template(model_name):
-    model_name = model_name.lower()
-    if "vicuna" in model_name or "output" in model_name:
-        return conv_vicuna_v1_1
-    elif "koala" in model_name:
-        return conv_koala_v1
-    elif "dolly-v2" in model_name:
-        return conv_dolly
-    elif "oasst" in model_name and "pythia" in model_name:
-        return conv_oasst
-    elif "baize" in model_name:
-        return conv_baize
-    elif "stablelm" in model_name:
-        return conv_stablelm
-    elif "rwkv-4" in model_name:
-        return conv_rwkv
-    return conv_one_shot
-
-
-if __name__ == "__main__":
-    conv = conv_templates["vicuna_v1.1"].copy()
-    conv.append_message(conv.roles[0], "Hello!")
-    conv.append_message(conv.roles[1], "Hi!")
-    conv.append_message(conv.roles[0], "How are you?")
-    conv.append_message(conv.roles[1], None)
-    print(conv.get_prompt())

lisa_on_cuda/utils/create_folders_and_variables_if_not_exists.py

@@ -1,56 +0,0 @@
-import json
-import logging
-import os
-from pathlib import Path
-
-
-def stats_pathname(pathname: Path | str):
-    current_pathname = Path(pathname)
-    return current_pathname.is_dir()
-
-
-def create_folder_if_not_exists(pathname: Path | str):
-    current_pathname = Path(pathname)
-    try:
-        print(f"Pathname exists? {current_pathname.exists()}, That's a folder? {current_pathname.is_dir()}...")
-        logging.info(f"Pathname exists? {current_pathname.exists()}, That's a folder? {current_pathname.is_dir()}...")
-        current_pathname.unlink(missing_ok=True)
-    except PermissionError as pe:
-        print(f"permission denied on removing pathname before folder creation:{pe}.")
-        logging.error(f"permission denied on removing pathname before folder creation:{pe}.")
-    except IsADirectoryError as errdir:
-        print(f"that's a directory:{errdir}.")
-        logging.error(f"that's a directory:{errdir}.")
-
-    print(f"Creating pathname: {current_pathname} ...")
-    logging.info(f"Creating pathname: {current_pathname} ...")
-    current_pathname.mkdir(mode=0o770, parents=True, exist_ok=True)
-
-    print(f"assertion: pathname exists and is a folder: {current_pathname} ...")
-    logging.info(f"assertion: pathname exists and is a folder: {current_pathname} ...")
-    assert current_pathname.is_dir()
-
-
-def folders_creation():
-    folders_string = os.getenv("FOLDERS_MAP")
-    try:
-        folders_dict = json.loads(folders_string)
-        for folder_env_ref, folder_env_path in folders_dict.items():
-            print(f"folder_env_ref:{folder_env_ref}, folder_env_path:{folder_env_path}.")
-            logging.info(f"folder_env_ref:{folder_env_ref}, folder_env_path:{folder_env_path}.")
-            create_folder_if_not_exists(folder_env_path)
-            print("========")
-            assert os.getenv(folder_env_ref) == folder_env_path
-    except (json.JSONDecodeError, TypeError) as jde:
-        print(f"jde:{jde}.")
-        logging.error(f"jde:{jde}.")
-        print("double check your variables, e.g. for misspelling like 'FOLDER_MAP'...")
-        logging.info("double check your variables, e.g. for misspelling like 'FOLDER_MAP' instead than 'FOLDERS_MAP'...")
-        for k_env, v_env in dict(os.environ).items():
-            print(f"{k_env}, v_env:{v_env}.")
-            logging.info(f"{k_env}, v_env:{v_env}.")
-
-
-if __name__ == '__main__':
-    folders_creation()
-

lisa_on_cuda/utils/data_processing.py

@@ -1,90 +0,0 @@
-import glob
-import json
-import os
-
-import cv2
-import numpy as np
-
-
-def get_mask_from_json(json_path, img):
-    try:
-        with open(json_path, "r") as r:
-            anno = json.loads(r.read())
-    except:
-        with open(json_path, "r", encoding="cp1252") as r:
-            anno = json.loads(r.read())
-
-    inform = anno["shapes"]
-    comments = anno["text"]
-    is_sentence = anno["is_sentence"]
-
-    height, width = img.shape[:2]
-
-    ### sort polies by area
-    area_list = []
-    valid_poly_list = []
-    for i in inform:
-        label_id = i["label"]
-        points = i["points"]
-        if "flag" == label_id.lower():  ## meaningless deprecated annotations
-            continue
-
-        tmp_mask = np.zeros((height, width), dtype=np.uint8)
-        cv2.polylines(tmp_mask, np.array([points], dtype=np.int32), True, 1, 1)
-        cv2.fillPoly(tmp_mask, np.array([points], dtype=np.int32), 1)
-        tmp_area = tmp_mask.sum()
-
-        area_list.append(tmp_area)
-        valid_poly_list.append(i)
-
-    ### ground-truth mask
-    sort_index = np.argsort(area_list)[::-1].astype(np.int32)
-    sort_index = list(sort_index)
-    sort_inform = []
-    for s_idx in sort_index:
-        sort_inform.append(valid_poly_list[s_idx])
-
-    mask = np.zeros((height, width), dtype=np.uint8)
-    for i in sort_inform:
-        label_id = i["label"]
-        points = i["points"]
-
-        if "ignore" in label_id.lower():
-            label_value = 255  # ignored during evaluation
-        else:
-            label_value = 1  # target
-
-        cv2.polylines(mask, np.array([points], dtype=np.int32), True, label_value, 1)
-        cv2.fillPoly(mask, np.array([points], dtype=np.int32), label_value)
-
-    return mask, comments, is_sentence
-
-
-if __name__ == "__main__":
-    data_dir = "./train"
-    vis_dir = "./vis"
-
-    if not os.path.exists(vis_dir):
-        os.makedirs(vis_dir)
-
-    json_path_list = sorted(glob.glob(data_dir + "/*.json"))
-    for json_path in json_path_list:
-        img_path = json_path.replace(".json", ".jpg")
-        img = cv2.imread(img_path)[:, :, ::-1]
-
-        # In generated mask, value 1 denotes valid target region, and value 255 stands for region ignored during evaluaiton.
-        mask, comments, is_sentence = get_mask_from_json(json_path, img)
-
-        ## visualization. Green for target, and red for ignore.
-        valid_mask = (mask == 1).astype(np.float32)[:, :, None]
-        ignore_mask = (mask == 255).astype(np.float32)[:, :, None]
-        vis_img = img * (1 - valid_mask) * (1 - ignore_mask) + (
-            (np.array([0, 255, 0]) * 0.6 + img * 0.4) * valid_mask
-            + (np.array([255, 0, 0]) * 0.6 + img * 0.4) * ignore_mask
-        )
-        vis_img = np.concatenate([img, vis_img], 1)
-        vis_path = os.path.join(
-            vis_dir, json_path.split("/")[-1].replace(".json", ".jpg")
-        )
-        cv2.imwrite(vis_path, vis_img[:, :, ::-1])
-        print("Visualization has been saved to: ", vis_path)

lisa_on_cuda/utils/dataset.py

@@ -1,466 +0,0 @@
-import glob
-import os
-import random
-
-import cv2
-import numpy as np
-import torch
-import torch.nn.functional as F
-from pycocotools import mask
-from transformers import CLIPImageProcessor
-
-from lisa_on_cuda.llava import conversation as conversation_lib
-from lisa_on_cuda.llava.constants import (DEFAULT_IMAGE_TOKEN, IGNORE_INDEX,
-                                   IMAGE_TOKEN_INDEX)
-from lisa_on_cuda.llava.mm_utils import tokenizer_image_token
-from lisa_on_cuda.segment_anything.utils.transforms import ResizeLongestSide
-
-from .conversation import get_default_conv_template
-from .data_processing import get_mask_from_json
-from .reason_seg_dataset import ReasonSegDataset
-from .refer import REFER
-from .refer_seg_dataset import ReferSegDataset
-from .sem_seg_dataset import SemSegDataset
-from .utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
-                    DEFAULT_IMAGE_TOKEN)
-from .vqa_dataset import VQADataset
-
-
-def collate_fn(
-    batch, tokenizer=None, conv_type="llava_v1", use_mm_start_end=True, local_rank=-1
-):
-    image_path_list = []
-    images_list = []
-    images_clip_list = []
-    conversation_list = []
-    masks_list = []
-    label_list = []
-    resize_list = []
-    questions_list = []
-    sampled_classes_list = []
-    offset_list = [0]
-    cnt = 0
-    inferences = []
-    for (
-        image_path,
-        images,
-        images_clip,
-        conversations,
-        masks,
-        label,
-        resize,
-        questions,
-        sampled_classes,
-        inference,
-    ) in batch:
-        image_path_list.append(image_path)
-        images_list.append(images)
-        images_clip_list.append(images_clip)
-        conversation_list.extend(conversations)
-        label_list.append(label)
-        masks_list.append(masks.float())
-        resize_list.append(resize)
-        questions_list.append(questions)
-        sampled_classes_list.append(sampled_classes)
-        cnt += len(conversations)
-        offset_list.append(cnt)
-        inferences.append(inference)
-
-    if use_mm_start_end:
-        # replace <image> token
-        for i in range(len(conversation_list)):
-            replace_token = DEFAULT_IMAGE_TOKEN
-            replace_token = (
-                DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN
-            )
-            conversation_list[i] = conversation_list[i].replace(
-                DEFAULT_IMAGE_TOKEN, replace_token
-            )
-    input_ids = [
-        tokenizer_image_token(prompt, tokenizer, return_tensors="pt")
-        for prompt in conversation_list
-    ]
-    input_ids = torch.nn.utils.rnn.pad_sequence(
-        input_ids, batch_first=True, padding_value=tokenizer.pad_token_id
-    )
-    attention_masks = input_ids.ne(tokenizer.pad_token_id)
-
-    conv = conversation_lib.default_conversation.copy()
-    targets = input_ids.clone()
-
-    if conv_type == "llava_v1":
-        sep = conv.sep + conv.roles[1] + ": "
-    else:
-        sep = "[/INST] "
-    for conversation, target in zip(conversation_list, targets):
-        total_len = int(target.ne(tokenizer.pad_token_id).sum())
-
-        rounds = conversation.split(conv.sep2)
-        cur_len = 1
-        target[:cur_len] = IGNORE_INDEX
-        for i, rou in enumerate(rounds):
-            if rou == "":
-                break
-
-            parts = rou.split(sep)
-            # if len(parts) != 2:
-            #     break
-            assert len(parts) == 2, (len(parts), rou)
-            parts[0] += sep
-
-            if DEFAULT_IMAGE_TOKEN in conversation:
-                round_len = len(tokenizer_image_token(rou, tokenizer))
-                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 2
-            else:
-                round_len = len(tokenizer(rou).input_ids)
-                instruction_len = len(tokenizer(parts[0]).input_ids) - 2
-
-            target[cur_len : cur_len + instruction_len] = IGNORE_INDEX
-
-            cur_len += round_len
-        target[cur_len:] = IGNORE_INDEX
-
-        if False:
-            z = target.clone()
-            z = torch.where(z == IGNORE_INDEX, tokenizer.unk_token_id, z)
-            if local_rank == 0:
-                print(
-                    "conversation: ",
-                    conversation,
-                    "tokenizer.decode(z): ",
-                    tokenizer.decode(z),
-                )
-
-        if cur_len < tokenizer.model_max_length:
-            assert cur_len == total_len
-
-    if inferences[0] == False:
-        truncate_len = tokenizer.model_max_length - 255
-
-        if input_ids.shape[1] > truncate_len:
-            input_ids = input_ids[:, :truncate_len]
-            targets = targets[:, :truncate_len]
-            attention_masks = attention_masks[:, :truncate_len]
-
-    return {
-        "image_paths": image_path_list,
-        "images": torch.stack(images_list, dim=0),
-        "images_clip": torch.stack(images_clip_list, dim=0),
-        "input_ids": input_ids,
-        "labels": targets,
-        "attention_masks": attention_masks,
-        "masks_list": masks_list,
-        "label_list": label_list,
-        "resize_list": resize_list,
-        "offset": torch.LongTensor(offset_list),
-        "questions_list": questions_list,
-        "sampled_classes_list": sampled_classes_list,
-        "inference": inferences[0],
-        "conversation_list": conversation_list,
-    }
-
-
-class HybridDataset(torch.utils.data.Dataset):
-    pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
-    pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
-    img_size = 1024
-    ignore_label = 255
-
-    def __init__(
-        self,
-        base_image_dir,
-        tokenizer,
-        vision_tower,
-        samples_per_epoch=500 * 8 * 2 * 10,
-        precision: str = "fp32",
-        image_size: int = 224,
-        num_classes_per_sample: int = 3,
-        exclude_val=False,
-        dataset="sem_seg||refer_seg||vqa||reason_seg",
-        sample_rate=[9, 3, 3, 1],
-        sem_seg_data="ade20k||cocostuff||partimagenet||pascal_part||paco_lvis||mapillary",
-        refer_seg_data="refclef||refcoco||refcoco+||refcocog",
-        vqa_data="llava_instruct_150k",
-        reason_seg_data="ReasonSeg|train",
-        explanatory=0.1,
-    ):
-        self.exclude_val = exclude_val
-        self.dataset = dataset
-        self.samples_per_epoch = samples_per_epoch
-        self.explanatory = explanatory
-        self.num_classes_per_sample = num_classes_per_sample
-        sample_rate = np.array(sample_rate)
-        self.sample_rate = sample_rate / sample_rate.sum()
-
-        self.base_image_dir = base_image_dir
-        self.image_size = image_size
-        self.tokenizer = tokenizer
-        self.precision = precision
-
-        self.datasets = dataset.split("||")
-
-        self.all_datasets = []
-        for dataset in self.datasets:
-            if dataset == "sem_seg":
-                self.all_datasets.append(
-                    SemSegDataset(
-                        base_image_dir,
-                        tokenizer,
-                        vision_tower,
-                        samples_per_epoch,
-                        precision,
-                        image_size,
-                        num_classes_per_sample,
-                        exclude_val,
-                        sem_seg_data,
-                    )
-                )
-            elif dataset == "refer_seg":
-                self.all_datasets.append(
-                    ReferSegDataset(
-                        base_image_dir,
-                        tokenizer,
-                        vision_tower,
-                        samples_per_epoch,
-                        precision,
-                        image_size,
-                        num_classes_per_sample,
-                        exclude_val,
-                        refer_seg_data,
-                    )
-                )
-            elif dataset == "vqa":
-                self.all_datasets.append(
-                    VQADataset(
-                        base_image_dir,
-                        tokenizer,
-                        vision_tower,
-                        samples_per_epoch,
-                        precision,
-                        image_size,
-                        num_classes_per_sample,
-                        exclude_val,
-                        vqa_data,
-                    )
-                )
-            elif dataset == "reason_seg":
-                self.all_datasets.append(
-                    ReasonSegDataset(
-                        base_image_dir,
-                        tokenizer,
-                        vision_tower,
-                        samples_per_epoch,
-                        precision,
-                        image_size,
-                        num_classes_per_sample,
-                        exclude_val,
-                        reason_seg_data,
-                        explanatory,
-                    )
-                )
-
-    def __len__(self):
-        return self.samples_per_epoch
-
-    def __getitem__(self, idx):
-        ind = np.random.choice(list(range(len(self.datasets))), p=self.sample_rate)
-        data = self.all_datasets[ind]
-        inference = False
-        return *data[0], inference
-
-
-class ValDataset(torch.utils.data.Dataset):
-    pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
-    pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
-    img_size = 1024
-    ignore_label = 255
-
-    def __init__(
-        self,
-        base_image_dir,
-        tokenizer,
-        vision_tower,
-        val_dataset,
-        image_size=1024,
-    ):
-        self.base_image_dir = base_image_dir
-        splits = val_dataset.split("|")
-        if len(splits) == 2:
-            ds, split = splits
-            images = glob.glob(
-                os.path.join(self.base_image_dir, "reason_seg", ds, split, "*.jpg")
-            )
-            self.images = images
-            self.data_type = "reason_seg"
-        elif len(splits) == 3:
-            ds, splitBy, split = splits
-            refer_api = REFER(self.base_image_dir, ds, splitBy)
-            ref_ids_val = refer_api.getRefIds(split=split)
-            images_ids_val = refer_api.getImgIds(ref_ids=ref_ids_val)
-            refs_val = refer_api.loadRefs(ref_ids=ref_ids_val)
-            refer_seg_ds = {}
-            refer_seg_ds["images"] = []
-            loaded_images = refer_api.loadImgs(image_ids=images_ids_val)
-            for item in loaded_images:
-                item = item.copy()
-                if ds == "refclef":
-                    item["file_name"] = os.path.join(
-                        base_image_dir, "images/saiapr_tc-12", item["file_name"]
-                    )
-                elif ds in ["refcoco", "refcoco+", "refcocog", "grefcoco"]:
-                    item["file_name"] = os.path.join(
-                        base_image_dir,
-                        "images/mscoco/images/train2014",
-                        item["file_name"],
-                    )
-                refer_seg_ds["images"].append(item)
-            refer_seg_ds["annotations"] = refer_api.Anns  # anns_val
-
-            img2refs = {}
-            for ref in refs_val:
-                image_id = ref["image_id"]
-                img2refs[image_id] = img2refs.get(image_id, []) + [
-                    ref,
-                ]
-            refer_seg_ds["img2refs"] = img2refs
-            self.refer_seg_ds = refer_seg_ds
-            self.data_type = "refer_seg"
-
-        self.ds = ds
-        self.image_size = image_size
-        self.tokenizer = tokenizer
-        self.transform = ResizeLongestSide(image_size)
-        self.clip_image_processor = CLIPImageProcessor.from_pretrained(vision_tower)
-
-    def __len__(self):
-        if self.data_type == "refer_seg":
-            return len(self.refer_seg_ds["images"])
-        else:
-            return len(self.images)
-
-    def preprocess(self, x: torch.Tensor) -> torch.Tensor:
-        """Normalize pixel values and pad to a square input."""
-        # Normalize colors
-        x = (x - self.pixel_mean) / self.pixel_std
-
-        # Pad
-        h, w = x.shape[-2:]
-        padh = self.img_size - h
-        padw = self.img_size - w
-        x = F.pad(x, (0, padw, 0, padh))
-        return x
-
-    def __getitem__(self, idx):
-        if self.data_type == "refer_seg":
-            refer_seg_ds = self.refer_seg_ds
-            images = refer_seg_ds["images"]
-            annotations = refer_seg_ds["annotations"]
-            img2refs = refer_seg_ds["img2refs"]
-
-            image_info = images[idx]
-            image_path = image_info["file_name"]
-            image_id = image_info["id"]
-
-            refs = img2refs[image_id]
-            if len(refs) == 0:
-                raise ValueError("image {} has no refs".format(image_id))
-
-            sents = []
-            ann_ids = []
-            for ref in refs:
-                for sent in ref["sentences"]:
-                    sents.append(sent["sent"].strip().lower())
-                    ann_ids.append(ref["ann_id"])
-
-            sampled_sents = sents
-            sampled_ann_ids = ann_ids
-            image = cv2.imread(image_path)
-            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-            is_sentence = False
-        else:
-            image_path = self.images[idx]
-            image = cv2.imread(image_path)
-            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-            json_path = image_path.replace(".jpg", ".json")
-            mask_json, sampled_sents, is_sentence = get_mask_from_json(json_path, image)
-            sampled_sents = [sampled_sents[0]]
-
-        conversations = []
-        conv = conversation_lib.default_conversation.copy()
-        i = 0
-        while i < len(sampled_sents):
-            conv.messages = []
-            text = sampled_sents[i].strip()
-            if is_sentence:
-                conv.append_message(
-                    conv.roles[0],
-                    DEFAULT_IMAGE_TOKEN
-                    + "\n {} Please output segmentation mask.".format(text),
-                )
-                conv.append_message(conv.roles[1], "[SEG].")
-            else:
-                conv.append_message(
-                    conv.roles[0],
-                    DEFAULT_IMAGE_TOKEN
-                    + "\n What is {} in this image? Please output segmentation mask.".format(
-                        text
-                    ),
-                )
-                conv.append_message(conv.roles[1], "[SEG].")
-            conversations.append(conv.get_prompt())
-            i += 1
-
-        # preprocess image for clip
-        image_clip = self.clip_image_processor.preprocess(image, return_tensors="pt")[
-            "pixel_values"
-        ][0]
-
-        # preprocess image for sam
-        image = self.transform.apply_image(image)
-        resize = image.shape[:2]
-        image = self.preprocess(torch.from_numpy(image).permute(2, 0, 1).contiguous())
-
-        if self.data_type == "refer_seg":
-            masks = []
-            for i, ann_id in enumerate(sampled_ann_ids):
-                ann = annotations[ann_id]
-                if len(ann["segmentation"]) == 0 and sampled_sents[i] != "":
-                    m = np.zeros((image_info["height"], image_info["width"], 1))
-                else:
-                    if type(ann["segmentation"][0]) == list:  # polygon
-                        rle = mask.frPyObjects(
-                            ann["segmentation"],
-                            image_info["height"],
-                            image_info["width"],
-                        )
-                    else:
-                        rle = ann["segmentation"]
-                        for i in range(len(rle)):
-                            if not isinstance(rle[i]["counts"], bytes):
-                                rle[i]["counts"] = rle[i]["counts"].encode()
-                    m = mask.decode(rle)
-                m = np.sum(
-                    m, axis=2
-                )  # sometimes there are multiple binary map (corresponding to multiple segs)
-                m = m.astype(np.uint8)  # convert to np.uint8
-                masks.append(m)
-        else:
-            masks = [mask_json]
-
-        masks = np.stack(masks, axis=0)
-        masks = torch.from_numpy(masks)
-        labels = torch.ones(masks.shape[1], masks.shape[2]) * self.ignore_label
-        inference = True
-
-        return (
-            image_path,
-            image,
-            image_clip,
-            conversations,
-            masks,
-            labels,
-            resize,
-            None,
-            None,
-            inference,
-        )

lisa_on_cuda/utils/frontend_builder.py

@@ -1,89 +0,0 @@
-import logging
-import os
-import subprocess
-from pathlib import Path
-
-from dotenv import load_dotenv
-
-from lisa_on_cuda.utils import session_logger
-
-
-load_dotenv()
-LOGLEVEL = os.getenv('LOGLEVEL', 'INFO').upper()
-session_logger.change_logging(LOGLEVEL)
-root_folder = Path(globals().get("__file__", "./_")).absolute().parent.parent.parent
-env_project_root_folder = Path(os.getenv("PROJECT_ROOT_FOLDER", root_folder))
-env_input_css_path = Path(os.getenv("INPUT_CSS_PATH"))
-
-
-def assert_envs(envs_list):
-    for current_env in envs_list:
-        try:
-            assert current_env is not None and current_env != ""
-        except AssertionError as aex:
-            logging.error(f"error on assertion for current_env: {current_env}.")
-            raise aex
-
-
-def read_std_out_err(std_out_err, output_type: str, command: list):
-    output = std_out_err.split("\n")
-    logging.info(f"output type:{output_type} for command:{' '.join(command)}.")
-    for line in iter(output):
-        logging.info(f"output_content_home stdout:{line.strip()}.")
-    logging.info("########")
-
-
-def run_command(commands_list: list, capture_output: bool = True, text: bool = True, check: bool = True) -> None:
-    try:
-        output_content_home = subprocess.run(
-            commands_list,
-            capture_output=capture_output,
-            text=text,
-            check=check
-        )
-        read_std_out_err(output_content_home.stdout, "stdout", commands_list)
-        read_std_out_err(output_content_home.stderr, "stderr", commands_list)
-    except Exception as ex:
-        logging.error(f"ex:{ex}.")
-        raise ex
-
-
-def build_frontend(
-        project_root_folder: str | Path,
-        input_css_path: str | Path,
-        output_dist_folder: Path = root_folder / "static" / "dist",
-    ) -> None:
-    assert_envs([
-        str(project_root_folder),
-        str(input_css_path)
-    ])
-
-    # install deps
-    os.chdir(Path(project_root_folder) / "static")
-    current_folder = os.getcwd()
-    logging.info(f"current_folder:{current_folder}, install pnpm...")
-    run_command(["npm", "install", "-g", "npm", "pnpm"])
-    logging.info(f"install pnpm dependencies...")
-    run_command(["pnpm", "install"])
-
-    # build frontend dist and assert for its correct build
-    output_css = str(output_dist_folder / "output.css")
-    output_index_html = str(output_dist_folder / "index.html")
-    output_dist_folder = str(output_dist_folder)
-    logging.info(f"pnpm: build '{output_dist_folder}'...")
-    run_command(["pnpm", "build"])
-    logging.info(f"pnpm: ls -l {output_index_html}:")
-    run_command(["ls", "-l", output_index_html])
-    cmd = ["pnpm", "tailwindcss", "-i", str(input_css_path), "-o", output_css]
-    logging.info(f"pnpm: {' '.join(cmd)}...")
-    run_command(["pnpm", "tailwindcss", "-i", str(input_css_path), "-o", output_css])
-    logging.info(f"pnpm: ls -l {output_css}:")
-    run_command(["ls", "-l", output_css])
-    logging.info(f"end!")
-
-
-if __name__ == '__main__':
-    build_frontend(
-        project_root_folder=env_project_root_folder,
-        input_css_path=env_input_css_path
-    )

lisa_on_cuda/utils/grefcoco.py

@@ -1,198 +0,0 @@
-import contextlib
-import copy
-import io
-import logging
-import os
-import random
-
-import numpy as np
-import pycocotools.mask as mask_util
-from detectron2.structures import Boxes, BoxMode, PolygonMasks, RotatedBoxes
-from detectron2.utils.file_io import PathManager
-from fvcore.common.timer import Timer
-from PIL import Image
-
-"""
-This file contains functions to parse RefCOCO-format annotations into dicts in "Detectron2 format".
-"""
-
-
-logger = logging.getLogger(__name__)
-
-__all__ = ["load_refcoco_json"]
-
-
-def load_grefcoco_json(
-    refer_root,
-    dataset_name,
-    splitby,
-    split,
-    image_root,
-    extra_annotation_keys=None,
-    extra_refer_keys=None,
-):
-    if dataset_name == "refcocop":
-        dataset_name = "refcoco+"
-    if dataset_name == "refcoco" or dataset_name == "refcoco+":
-        splitby == "unc"
-    if dataset_name == "refcocog":
-        assert splitby == "umd" or splitby == "google"
-
-    dataset_id = "_".join([dataset_name, splitby, split])
-
-    from .grefer import G_REFER
-
-    logger.info("Loading dataset {} ({}-{}) ...".format(dataset_name, splitby, split))
-    logger.info("Refcoco root: {}".format(refer_root))
-    timer = Timer()
-    refer_root = PathManager.get_local_path(refer_root)
-    with contextlib.redirect_stdout(io.StringIO()):
-        refer_api = G_REFER(data_root=refer_root, dataset=dataset_name, splitBy=splitby)
-    if timer.seconds() > 1:
-        logger.info(
-            "Loading {} takes {:.2f} seconds.".format(dataset_id, timer.seconds())
-        )
-
-    ref_ids = refer_api.getRefIds(split=split)
-    img_ids = refer_api.getImgIds(ref_ids)
-    refs = refer_api.loadRefs(ref_ids)
-    imgs = [refer_api.loadImgs(ref["image_id"])[0] for ref in refs]
-    anns = [refer_api.loadAnns(ref["ann_id"]) for ref in refs]
-    imgs_refs_anns = list(zip(imgs, refs, anns))
-
-    logger.info(
-        "Loaded {} images, {} referring object sets in G_RefCOCO format from {}".format(
-            len(img_ids), len(ref_ids), dataset_id
-        )
-    )
-
-    dataset_dicts = []
-
-    ann_keys = ["iscrowd", "bbox", "category_id"] + (extra_annotation_keys or [])
-    ref_keys = ["raw", "sent_id"] + (extra_refer_keys or [])
-
-    ann_lib = {}
-
-    NT_count = 0
-    MT_count = 0
-
-    for img_dict, ref_dict, anno_dicts in imgs_refs_anns:
-        record = {}
-        record["source"] = "grefcoco"
-        record["file_name"] = os.path.join(image_root, img_dict["file_name"])
-        record["height"] = img_dict["height"]
-        record["width"] = img_dict["width"]
-        image_id = record["image_id"] = img_dict["id"]
-
-        # Check that information of image, ann and ref match each other
-        # This fails only when the data parsing logic or the annotation file is buggy.
-        assert ref_dict["image_id"] == image_id
-        assert ref_dict["split"] == split
-        if not isinstance(ref_dict["ann_id"], list):
-            ref_dict["ann_id"] = [ref_dict["ann_id"]]
-
-        # No target samples
-        if None in anno_dicts:
-            assert anno_dicts == [None]
-            assert ref_dict["ann_id"] == [-1]
-            record["empty"] = True
-            obj = {key: None for key in ann_keys if key in ann_keys}
-            obj["bbox_mode"] = BoxMode.XYWH_ABS
-            obj["empty"] = True
-            obj = [obj]
-
-        # Multi target samples
-        else:
-            record["empty"] = False
-            obj = []
-            for anno_dict in anno_dicts:
-                ann_id = anno_dict["id"]
-                if anno_dict["iscrowd"]:
-                    continue
-                assert anno_dict["image_id"] == image_id
-                assert ann_id in ref_dict["ann_id"]
-
-                if ann_id in ann_lib:
-                    ann = ann_lib[ann_id]
-                else:
-                    ann = {key: anno_dict[key] for key in ann_keys if key in anno_dict}
-                    ann["bbox_mode"] = BoxMode.XYWH_ABS
-                    ann["empty"] = False
-
-                    segm = anno_dict.get("segmentation", None)
-                    assert segm  # either list[list[float]] or dict(RLE)
-                    if isinstance(segm, dict):
-                        if isinstance(segm["counts"], list):
-                            # convert to compressed RLE
-                            segm = mask_util.frPyObjects(segm, *segm["size"])
-                    else:
-                        # filter out invalid polygons (< 3 points)
-                        segm = [
-                            poly
-                            for poly in segm
-                            if len(poly) % 2 == 0 and len(poly) >= 6
-                        ]
-                        if len(segm) == 0:
-                            num_instances_without_valid_segmentation += 1
-                            continue  # ignore this instance
-                    ann["segmentation"] = segm
-                    ann_lib[ann_id] = ann
-
-                obj.append(ann)
-
-        record["annotations"] = obj
-
-        # Process referring expressions
-        sents = ref_dict["sentences"]
-        for sent in sents:
-            ref_record = record.copy()
-            ref = {key: sent[key] for key in ref_keys if key in sent}
-            ref["ref_id"] = ref_dict["ref_id"]
-            ref_record["sentence"] = ref
-            dataset_dicts.append(ref_record)
-    #         if ref_record['empty']:
-    #             NT_count += 1
-    #         else:
-    #             MT_count += 1
-
-    # logger.info("NT samples: %d, MT samples: %d", NT_count, MT_count)
-
-    # Debug mode
-    # return dataset_dicts[:100]
-
-    return dataset_dicts
-
-
-if __name__ == "__main__":
-    """
-    Test the COCO json dataset loader.
-
-    Usage:
-        python -m detectron2.data.datasets.coco \
-            path/to/json path/to/image_root dataset_name
-
-        "dataset_name" can be "coco_2014_minival_100", or other
-        pre-registered ones
-    """
-    import sys
-
-    import detectron2.data.datasets  # noqa # add pre-defined metadata
-    from detectron2.utils.logger import setup_logger
-    from detectron2.utils.visualizer import Visualizer
-
-    REFCOCO_PATH = "/mnt/lustre/hhding/code/ReLA/datasets"
-    COCO_TRAIN_2014_IMAGE_ROOT = "/mnt/lustre/hhding/code/ReLA/datasets/images"
-    REFCOCO_DATASET = "grefcoco"
-    REFCOCO_SPLITBY = "unc"
-    REFCOCO_SPLIT = "train"
-
-    logger = setup_logger(name=__name__)
-
-    dicts = load_grefcoco_json(
-        REFCOCO_PATH,
-        REFCOCO_DATASET,
-        REFCOCO_SPLITBY,
-        REFCOCO_SPLIT,
-        COCO_TRAIN_2014_IMAGE_ROOT,
-    )
-    logger.info("Done loading {} samples.".format(len(dicts)))

lisa_on_cuda/utils/grefer.py

@@ -1,352 +0,0 @@
-"""
-grefer v0.1
-This interface provides access to gRefCOCO.
-
-The following API functions are defined:
-G_REFER      - REFER api class
-getRefIds    - get ref ids that satisfy given filter conditions.
-getAnnIds    - get ann ids that satisfy given filter conditions.
-getImgIds    - get image ids that satisfy given filter conditions.
-getCatIds    - get category ids that satisfy given filter conditions.
-loadRefs     - load refs with the specified ref ids.
-loadAnns     - load anns with the specified ann ids.
-loadImgs     - load images with the specified image ids.
-loadCats     - load category names with the specified category ids.
-getRefBox    - get ref's bounding box [x, y, w, h] given the ref_id
-showRef      - show image, segmentation or box of the referred object with the ref
-getMaskByRef - get mask and area of the referred object given ref or ref ids
-getMask      - get mask and area of the referred object given ref
-showMask     - show mask of the referred object given ref
-"""
-
-import itertools
-import json
-import os.path as osp
-import pickle
-import time
-
-import matplotlib.pyplot as plt
-import numpy as np
-import skimage.io as io
-from matplotlib.collections import PatchCollection
-from matplotlib.patches import Polygon, Rectangle
-from pycocotools import mask
-
-
-class G_REFER:
-    def __init__(self, data_root, dataset="grefcoco", splitBy="unc"):
-        # provide data_root folder which contains grefcoco
-        print("loading dataset %s into memory..." % dataset)
-        self.ROOT_DIR = osp.abspath(osp.dirname(__file__))
-        self.DATA_DIR = osp.join(data_root, dataset)
-        if dataset in ["grefcoco"]:
-            self.IMAGE_DIR = osp.join(data_root, "images/train2014")
-        else:
-            raise KeyError("No refer dataset is called [%s]" % dataset)
-
-        tic = time.time()
-
-        # load refs from data/dataset/refs(dataset).json
-        self.data = {}
-        self.data["dataset"] = dataset
-
-        ref_file = osp.join(self.DATA_DIR, f"grefs({splitBy}).p")
-        if osp.exists(ref_file):
-            self.data["refs"] = pickle.load(open(ref_file, "rb"), fix_imports=True)
-        else:
-            ref_file = osp.join(self.DATA_DIR, f"grefs({splitBy}).json")
-            if osp.exists(ref_file):
-                self.data["refs"] = json.load(open(ref_file, "rb"))
-            else:
-                raise FileNotFoundError("JSON file not found")
-
-        # load annotations from data/dataset/instances.json
-        instances_file = osp.join(self.DATA_DIR, "instances.json")
-        instances = json.load(open(instances_file, "r"))
-        self.data["images"] = instances["images"]
-        self.data["annotations"] = instances["annotations"]
-        self.data["categories"] = instances["categories"]
-
-        # create index
-        self.createIndex()
-        print("DONE (t=%.2fs)" % (time.time() - tic))
-
-    @staticmethod
-    def _toList(x):
-        return x if isinstance(x, list) else [x]
-
-    @staticmethod
-    def match_any(a, b):
-        a = a if isinstance(a, list) else [a]
-        b = b if isinstance(b, list) else [b]
-        return set(a) & set(b)
-
-    def createIndex(self):
-        # create sets of mapping
-        # 1)  Refs: 	 	{ref_id: ref}
-        # 2)  Anns: 	 	{ann_id: ann}
-        # 3)  Imgs:		 	{image_id: image}
-        # 4)  Cats: 	 	{category_id: category_name}
-        # 5)  Sents:     	{sent_id: sent}
-        # 6)  imgToRefs: 	{image_id: refs}
-        # 7)  imgToAnns: 	{image_id: anns}
-        # 8)  refToAnn:  	{ref_id: ann}
-        # 9)  annToRef:  	{ann_id: ref}
-        # 10) catToRefs: 	{category_id: refs}
-        # 11) sentToRef: 	{sent_id: ref}
-        # 12) sentToTokens: {sent_id: tokens}
-        print("creating index...")
-        # fetch info from instances
-        Anns, Imgs, Cats, imgToAnns = {}, {}, {}, {}
-        Anns[-1] = None
-        for ann in self.data["annotations"]:
-            Anns[ann["id"]] = ann
-            imgToAnns[ann["image_id"]] = imgToAnns.get(ann["image_id"], []) + [ann]
-        for img in self.data["images"]:
-            Imgs[img["id"]] = img
-        for cat in self.data["categories"]:
-            Cats[cat["id"]] = cat["name"]
-
-        # fetch info from refs
-        Refs, imgToRefs, refToAnn, annToRef, catToRefs = {}, {}, {}, {}, {}
-        Sents, sentToRef, sentToTokens = {}, {}, {}
-        availableSplits = []
-        for ref in self.data["refs"]:
-            # ids
-            ref_id = ref["ref_id"]
-            ann_id = ref["ann_id"]
-            category_id = ref["category_id"]
-            image_id = ref["image_id"]
-
-            if ref["split"] not in availableSplits:
-                availableSplits.append(ref["split"])
-
-            # add mapping related to ref
-            if ref_id in Refs:
-                print("Duplicate ref id")
-            Refs[ref_id] = ref
-            imgToRefs[image_id] = imgToRefs.get(image_id, []) + [ref]
-
-            category_id = self._toList(category_id)
-            added_cats = []
-            for cat in category_id:
-                if cat not in added_cats:
-                    added_cats.append(cat)
-                    catToRefs[cat] = catToRefs.get(cat, []) + [ref]
-
-            ann_id = self._toList(ann_id)
-            refToAnn[ref_id] = [Anns[ann] for ann in ann_id]
-            for ann_id_n in ann_id:
-                annToRef[ann_id_n] = annToRef.get(ann_id_n, []) + [ref]
-
-            # add mapping of sent
-            for sent in ref["sentences"]:
-                Sents[sent["sent_id"]] = sent
-                sentToRef[sent["sent_id"]] = ref
-                sentToTokens[sent["sent_id"]] = sent["tokens"]
-
-        # create class members
-        self.Refs = Refs
-        self.Anns = Anns
-        self.Imgs = Imgs
-        self.Cats = Cats
-        self.Sents = Sents
-        self.imgToRefs = imgToRefs
-        self.imgToAnns = imgToAnns
-        self.refToAnn = refToAnn
-        self.annToRef = annToRef
-        self.catToRefs = catToRefs
-        self.sentToRef = sentToRef
-        self.sentToTokens = sentToTokens
-        self.availableSplits = availableSplits
-        print("index created.")
-
-    def getRefIds(self, image_ids=[], cat_ids=[], split=[]):
-        image_ids = self._toList(image_ids)
-        cat_ids = self._toList(cat_ids)
-        split = self._toList(split)
-
-        for s in split:
-            if s not in self.availableSplits:
-                raise ValueError(f"Invalid split name: {s}")
-
-        refs = self.data["refs"]
-
-        if len(image_ids) > 0:
-            lists = [self.imgToRefs[image_id] for image_id in image_ids]
-            refs = list(itertools.chain.from_iterable(lists))
-        if len(cat_ids) > 0:
-            refs = [ref for ref in refs if self.match_any(ref["category_id"], cat_ids)]
-        if len(split) > 0:
-            refs = [ref for ref in refs if ref["split"] in split]
-
-        ref_ids = [ref["ref_id"] for ref in refs]
-        return ref_ids
-
-    def getAnnIds(self, image_ids=[], ref_ids=[]):
-        image_ids = self._toList(image_ids)
-        ref_ids = self._toList(ref_ids)
-
-        if any([len(image_ids), len(ref_ids)]):
-            if len(image_ids) > 0:
-                lists = [
-                    self.imgToAnns[image_id]
-                    for image_id in image_ids
-                    if image_id in self.imgToAnns
-                ]
-                anns = list(itertools.chain.from_iterable(lists))
-            else:
-                anns = self.data["annotations"]
-            ann_ids = [ann["id"] for ann in anns]
-            if len(ref_ids) > 0:
-                lists = [self.Refs[ref_id]["ann_id"] for ref_id in ref_ids]
-                anns_by_ref_id = list(itertools.chain.from_iterable(lists))
-                ann_ids = list(set(ann_ids).intersection(set(anns_by_ref_id)))
-        else:
-            ann_ids = [ann["id"] for ann in self.data["annotations"]]
-
-        return ann_ids
-
-    def getImgIds(self, ref_ids=[]):
-        ref_ids = self._toList(ref_ids)
-
-        if len(ref_ids) > 0:
-            image_ids = list(set([self.Refs[ref_id]["image_id"] for ref_id in ref_ids]))
-        else:
-            image_ids = self.Imgs.keys()
-        return image_ids
-
-    def getCatIds(self):
-        return self.Cats.keys()
-
-    def loadRefs(self, ref_ids=[]):
-        return [self.Refs[ref_id] for ref_id in self._toList(ref_ids)]
-
-    def loadAnns(self, ann_ids=[]):
-        if isinstance(ann_ids, str):
-            ann_ids = int(ann_ids)
-        return [self.Anns[ann_id] for ann_id in self._toList(ann_ids)]
-
-    def loadImgs(self, image_ids=[]):
-        return [self.Imgs[image_id] for image_id in self._toList(image_ids)]
-
-    def loadCats(self, cat_ids=[]):
-        return [self.Cats[cat_id] for cat_id in self._toList(cat_ids)]
-
-    def getRefBox(self, ref_id):
-        anns = self.refToAnn[ref_id]
-        return [ann["bbox"] for ann in anns]  # [x, y, w, h]
-
-    def showRef(self, ref, seg_box="seg"):
-        ax = plt.gca()
-        # show image
-        image = self.Imgs[ref["image_id"]]
-        I = io.imread(osp.join(self.IMAGE_DIR, image["file_name"]))
-        ax.imshow(I)
-        # show refer expression
-        for sid, sent in enumerate(ref["sentences"]):
-            print("%s. %s" % (sid + 1, sent["sent"]))
-        # show segmentations
-        if seg_box == "seg":
-            ann_id = ref["ann_id"]
-            ann = self.Anns[ann_id]
-            polygons = []
-            color = []
-            c = "none"
-            if type(ann["segmentation"][0]) == list:
-                # polygon used for refcoco*
-                for seg in ann["segmentation"]:
-                    poly = np.array(seg).reshape((len(seg) / 2, 2))
-                    polygons.append(Polygon(poly, True, alpha=0.4))
-                    color.append(c)
-                p = PatchCollection(
-                    polygons,
-                    facecolors=color,
-                    edgecolors=(1, 1, 0, 0),
-                    linewidths=3,
-                    alpha=1,
-                )
-                ax.add_collection(p)  # thick yellow polygon
-                p = PatchCollection(
-                    polygons,
-                    facecolors=color,
-                    edgecolors=(1, 0, 0, 0),
-                    linewidths=1,
-                    alpha=1,
-                )
-                ax.add_collection(p)  # thin red polygon
-            else:
-                # mask used for refclef
-                rle = ann["segmentation"]
-                m = mask.decode(rle)
-                img = np.ones((m.shape[0], m.shape[1], 3))
-                color_mask = np.array([2.0, 166.0, 101.0]) / 255
-                for i in range(3):
-                    img[:, :, i] = color_mask[i]
-                ax.imshow(np.dstack((img, m * 0.5)))
-        # show bounding-box
-        elif seg_box == "box":
-            ann_id = ref["ann_id"]
-            ann = self.Anns[ann_id]
-            bbox = self.getRefBox(ref["ref_id"])
-            box_plot = Rectangle(
-                (bbox[0], bbox[1]),
-                bbox[2],
-                bbox[3],
-                fill=False,
-                edgecolor="green",
-                linewidth=3,
-            )
-            ax.add_patch(box_plot)
-
-    def getMask(self, ann):
-        if not ann:
-            return None
-        if ann["iscrowd"]:
-            raise ValueError("Crowd object")
-        image = self.Imgs[ann["image_id"]]
-        if type(ann["segmentation"][0]) == list:  # polygon
-            rle = mask.frPyObjects(ann["segmentation"], image["height"], image["width"])
-        else:
-            rle = ann["segmentation"]
-
-        m = mask.decode(rle)
-        m = np.sum(
-            m, axis=2
-        )  # sometimes there are multiple binary map (corresponding to multiple segs)
-        m = m.astype(np.uint8)  # convert to np.uint8
-        # compute area
-        area = sum(mask.area(rle))  # should be close to ann['area']
-        return {"mask": m, "area": area}
-
-    def getMaskByRef(self, ref=None, ref_id=None, merge=False):
-        if not ref and not ref_id:
-            raise ValueError
-        if ref:
-            ann_ids = ref["ann_id"]
-            ref_id = ref["ref_id"]
-        else:
-            ann_ids = self.getAnnIds(ref_ids=ref_id)
-
-        if ann_ids == [-1]:
-            img = self.Imgs[self.Refs[ref_id]["image_id"]]
-            return {
-                "mask": np.zeros([img["height"], img["width"]], dtype=np.uint8),
-                "empty": True,
-            }
-
-        anns = self.loadAnns(ann_ids)
-        mask_list = [self.getMask(ann) for ann in anns if not ann["iscrowd"]]
-
-        if merge:
-            merged_masks = sum([mask["mask"] for mask in mask_list])
-            merged_masks[np.where(merged_masks > 1)] = 1
-            return {"mask": merged_masks, "empty": False}
-        else:
-            return mask_list
-
-    def showMask(self, ref):
-        M = self.getMask(ref)
-        msk = M["mask"]
-        ax = plt.gca()
-        ax.imshow(msk)

lisa_on_cuda/utils/reason_seg_dataset.py

@@ -1,218 +0,0 @@
-import glob
-import json
-import os
-import random
-
-import cv2
-import numpy as np
-import torch
-import torch.nn.functional as F
-from transformers import CLIPImageProcessor
-
-from lisa_on_cuda.llava import conversation as conversation_lib
-from lisa_on_cuda.segment_anything.utils.transforms import ResizeLongestSide
-
-from .data_processing import get_mask_from_json
-from .utils import (ANSWER_LIST, DEFAULT_IMAGE_TOKEN,
-                    EXPLANATORY_QUESTION_LIST, LONG_QUESTION_LIST,
-                    SHORT_QUESTION_LIST)
-
-
-class ReasonSegDataset(torch.utils.data.Dataset):
-    pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
-    pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
-    img_size = 1024
-    ignore_label = 255
-
-    def __init__(
-        self,
-        base_image_dir,
-        tokenizer,
-        vision_tower,
-        samples_per_epoch=500 * 8 * 2 * 10,
-        precision: str = "fp32",
-        image_size: int = 224,
-        num_classes_per_sample: int = 3,
-        exclude_val=False,
-        reason_seg_data="ReasonSeg|train",
-        explanatory=0.1,
-    ):
-        self.exclude_val = exclude_val
-        self.reason_seg_data = reason_seg_data
-        self.samples_per_epoch = samples_per_epoch
-        self.explanatory = explanatory
-        self.num_classes_per_sample = num_classes_per_sample
-
-        self.base_image_dir = base_image_dir
-        self.image_size = image_size
-        self.tokenizer = tokenizer
-        self.precision = precision
-        self.transform = ResizeLongestSide(image_size)
-        self.clip_image_processor = CLIPImageProcessor.from_pretrained(vision_tower)
-
-        self.short_question_list = SHORT_QUESTION_LIST
-        self.long_question_list = LONG_QUESTION_LIST
-        self.answer_list = ANSWER_LIST
-
-        reason_seg_data, splits = reason_seg_data.split("|")
-        splits = splits.split("_")
-        images = []
-        for split in splits:
-            images_split = glob.glob(
-                os.path.join(
-                    base_image_dir, "reason_seg", reason_seg_data, split, "*.jpg"
-                )
-            )
-            images.extend(images_split)
-        jsons = [path.replace(".jpg", ".json") for path in images]
-        self.reason_seg_data = (images, jsons)
-
-        print("number of reason_seg samples: ", len(images))
-
-        if explanatory != -1:
-            self.explanatory_question_list = EXPLANATORY_QUESTION_LIST
-            self.img_to_explanation = {}
-            with open(
-                os.path.join(
-                    base_image_dir,
-                    "reason_seg",
-                    reason_seg_data,
-                    "explanatory",
-                    "train.json",
-                )
-            ) as f:
-                items = json.load(f)
-            for item in items:
-                img_name = item["image"]
-                self.img_to_explanation[img_name] = {
-                    "query": item["query"],
-                    "outputs": item["outputs"],
-                }
-
-            print("len(self.img_to_explanation): ", len(self.img_to_explanation))
-
-    def __len__(self):
-        return self.samples_per_epoch
-
-    def preprocess(self, x: torch.Tensor) -> torch.Tensor:
-        """Normalize pixel values and pad to a square input."""
-        # Normalize colors
-        x = (x - self.pixel_mean) / self.pixel_std
-
-        # Pad
-        h, w = x.shape[-2:]
-        padh = self.img_size - h
-        padw = self.img_size - w
-        x = F.pad(x, (0, padw, 0, padh))
-        return x
-
-    def __getitem__(self, idx):
-        images, jsons = self.reason_seg_data
-        idx = random.randint(0, len(images) - 1)
-        image_path = images[idx]
-        json_path = jsons[idx]
-
-        image = cv2.imread(image_path)
-        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-        ori_size = image.shape[:2]
-        # preprocess image for clip
-        image_clip = self.clip_image_processor.preprocess(image, return_tensors="pt")[
-            "pixel_values"
-        ][0]
-
-        mask, sents, is_sentence = get_mask_from_json(json_path, image)
-        if len(sents) >= self.num_classes_per_sample:
-            sampled_inds = np.random.choice(
-                list(range(len(sents))), size=self.num_classes_per_sample, replace=False
-            )
-        else:
-            sampled_inds = list(range(len(sents)))
-        sampled_sents = np.vectorize(sents.__getitem__)(sampled_inds).tolist()
-        sampled_masks = [
-            (mask == 1).astype(np.float32) for _ in range(len(sampled_inds))
-        ]
-
-        image = self.transform.apply_image(image)  # preprocess image for sam
-        resize = image.shape[:2]
-
-        image_name = image_path.split("/")[-1]
-        if self.explanatory != -1 and image_name in self.img_to_explanation:
-            if random.random() < self.explanatory:
-                choice = 2
-            else:
-                choice = random.randint(0, 1)
-
-        questions = []
-        answers = []
-        for text in sampled_sents:
-            if is_sentence:
-                question_template = random.choice(self.long_question_list)
-                questions.append(question_template.format(sent=text))
-            else:
-                question_template = random.choice(self.short_question_list)
-                questions.append(question_template.format(class_name=text.lower()))
-
-            # add explanation if applicable
-            img_name = image_path.split("/")[-1]
-            if self.explanatory != -1 and img_name in self.img_to_explanation:
-                if choice == 0:  # [SEG] token
-                    answers.append(random.choice(self.answer_list))
-                elif choice == 1:  # [SEG] token + text answer
-                    image_name = image_path.split("/")[-1]
-                    answer = self.img_to_explanation[image_name]["outputs"]
-                    answer = random.choice(self.answer_list) + " {}".format(answer)
-                    questions[-1] = (
-                        DEFAULT_IMAGE_TOKEN
-                        + "\n"
-                        + text
-                        + " {}".format(random.choice(self.explanatory_question_list))
-                    )
-                    answers.append(answer)
-                elif choice == 2:  # vanilla text answer
-                    image_name = image_path.split("/")[-1]
-                    answer = self.img_to_explanation[image_name]["outputs"]
-                    questions[-1] = DEFAULT_IMAGE_TOKEN + "\n" + text
-                    answers.append(answer)
-                else:
-                    raise ValueError("Not implemented yet.")
-            else:
-                answers.append(random.choice(self.answer_list))
-
-            conversations = []
-            conv = conversation_lib.default_conversation.copy()
-            roles = {"human": conv.roles[0], "gpt": conv.roles[1]}
-
-            i = 0
-            while i < len(questions):
-                conv.messages = []
-                conv.append_message(conv.roles[0], questions[i])
-                conv.append_message(conv.roles[1], answers[i])
-                conversations.append(conv.get_prompt())
-                i += 1
-
-        image = self.preprocess(torch.from_numpy(image).permute(2, 0, 1).contiguous())
-
-        image_name = image_path.split("/")[-1]
-        if (
-            self.explanatory != -1
-            and image_name in self.img_to_explanation
-            and choice == 2
-        ):
-            masks = torch.rand(0, *ori_size)
-            label = torch.ones(ori_size) * self.ignore_label
-        else:
-            masks = np.stack(sampled_masks, axis=0)
-            masks = torch.from_numpy(masks)
-            label = torch.ones(masks.shape[1], masks.shape[2]) * self.ignore_label
-
-        return (
-            image_path,
-            image,
-            image_clip,
-            conversations,
-            masks,
-            label,
-            resize,
-            questions,
-            sampled_sents,
-        )

lisa_on_cuda/utils/refer.py

@@ -1,391 +0,0 @@
-__author__ = "licheng"
-
-"""
-This interface provides access to four datasets:
-1) refclef
-2) refcoco
-3) refcoco+
-4) refcocog
-split by unc and google
-
-The following API functions are defined:
-REFER      - REFER api class
-getRefIds  - get ref ids that satisfy given filter conditions.
-getAnnIds  - get ann ids that satisfy given filter conditions.
-getImgIds  - get image ids that satisfy given filter conditions.
-getCatIds  - get category ids that satisfy given filter conditions.
-loadRefs   - load refs with the specified ref ids.
-loadAnns   - load anns with the specified ann ids.
-loadImgs   - load images with the specified image ids.
-loadCats   - load category names with the specified category ids.
-getRefBox  - get ref's bounding box [x, y, w, h] given the ref_id
-showRef    - show image, segmentation or box of the referred object with the ref
-getMask    - get mask and area of the referred object given ref
-showMask   - show mask of the referred object given ref
-"""
-
-import itertools
-import json
-import os.path as osp
-import pickle
-import sys
-import time
-from pprint import pprint
-
-import matplotlib.pyplot as plt
-import numpy as np
-import skimage.io as io
-from matplotlib.collections import PatchCollection
-from matplotlib.patches import Polygon, Rectangle
-from pycocotools import mask
-
-
-class REFER:
-    def __init__(self, data_root, dataset="refcoco", splitBy="unc"):
-        # provide data_root folder which contains refclef, refcoco, refcoco+ and refcocog
-        # also provide dataset name and splitBy information
-        # e.g., dataset = 'refcoco', splitBy = 'unc'
-        print("loading dataset %s into memory..." % dataset)
-        self.ROOT_DIR = osp.abspath(osp.dirname(__file__))
-        self.DATA_DIR = osp.join(data_root, dataset)
-        if dataset in ["refcoco", "refcoco+", "refcocog"]:
-            self.IMAGE_DIR = osp.join(data_root, "images/mscoco/images/train2014")
-        elif dataset == "refclef":
-            self.IMAGE_DIR = osp.join(data_root, "images/saiapr_tc-12")
-        else:
-            print("No refer dataset is called [%s]" % dataset)
-            sys.exit()
-
-        self.dataset = dataset
-
-        # load refs from data/dataset/refs(dataset).json
-        tic = time.time()
-
-        ref_file = osp.join(self.DATA_DIR, "refs(" + splitBy + ").p")
-        print("ref_file: ", ref_file)
-        self.data = {}
-        self.data["dataset"] = dataset
-        self.data["refs"] = pickle.load(open(ref_file, "rb"))
-
-        # load annotations from data/dataset/instances.json
-        instances_file = osp.join(self.DATA_DIR, "instances.json")
-        instances = json.load(open(instances_file, "rb"))
-        self.data["images"] = instances["images"]
-        self.data["annotations"] = instances["annotations"]
-        self.data["categories"] = instances["categories"]
-
-        # create index
-        self.createIndex()
-        print("DONE (t=%.2fs)" % (time.time() - tic))
-
-    def createIndex(self):
-        # create sets of mapping
-        # 1)  Refs: 	 	{ref_id: ref}
-        # 2)  Anns: 	 	{ann_id: ann}
-        # 3)  Imgs:		 	{image_id: image}
-        # 4)  Cats: 	 	{category_id: category_name}
-        # 5)  Sents:     	{sent_id: sent}
-        # 6)  imgToRefs: 	{image_id: refs}
-        # 7)  imgToAnns: 	{image_id: anns}
-        # 8)  refToAnn:  	{ref_id: ann}
-        # 9)  annToRef:  	{ann_id: ref}
-        # 10) catToRefs: 	{category_id: refs}
-        # 11) sentToRef: 	{sent_id: ref}
-        # 12) sentToTokens: {sent_id: tokens}
-        print("creating index...")
-        # fetch info from instances
-        Anns, Imgs, Cats, imgToAnns = {}, {}, {}, {}
-        for ann in self.data["annotations"]:
-            Anns[ann["id"]] = ann
-            imgToAnns[ann["image_id"]] = imgToAnns.get(ann["image_id"], []) + [ann]
-        for img in self.data["images"]:
-            Imgs[img["id"]] = img
-        for cat in self.data["categories"]:
-            Cats[cat["id"]] = cat["name"]
-
-        # fetch info from refs
-        Refs, imgToRefs, refToAnn, annToRef, catToRefs = {}, {}, {}, {}, {}
-        Sents, sentToRef, sentToTokens = {}, {}, {}
-        for ref in self.data["refs"]:
-            # ids
-            ref_id = ref["ref_id"]
-            ann_id = ref["ann_id"]
-            category_id = ref["category_id"]
-            image_id = ref["image_id"]
-
-            # add mapping related to ref
-            Refs[ref_id] = ref
-            imgToRefs[image_id] = imgToRefs.get(image_id, []) + [ref]
-            catToRefs[category_id] = catToRefs.get(category_id, []) + [ref]
-            refToAnn[ref_id] = Anns[ann_id]
-            annToRef[ann_id] = ref
-
-            # add mapping of sent
-            for sent in ref["sentences"]:
-                Sents[sent["sent_id"]] = sent
-                sentToRef[sent["sent_id"]] = ref
-                sentToTokens[sent["sent_id"]] = sent["tokens"]
-
-        # create class members
-        self.Refs = Refs
-        self.Anns = Anns
-        self.Imgs = Imgs
-        self.Cats = Cats
-        self.Sents = Sents
-        self.imgToRefs = imgToRefs
-        self.imgToAnns = imgToAnns
-        self.refToAnn = refToAnn
-        self.annToRef = annToRef
-        self.catToRefs = catToRefs
-        self.sentToRef = sentToRef
-        self.sentToTokens = sentToTokens
-        print("index created.")
-
-    def getRefIds(self, image_ids=[], cat_ids=[], ref_ids=[], split=""):
-        image_ids = image_ids if type(image_ids) == list else [image_ids]
-        cat_ids = cat_ids if type(cat_ids) == list else [cat_ids]
-        ref_ids = ref_ids if type(ref_ids) == list else [ref_ids]
-
-        if len(image_ids) == len(cat_ids) == len(ref_ids) == len(split) == 0:
-            refs = self.data["refs"]
-        else:
-            if not len(image_ids) == 0:
-                refs = [self.imgToRefs[image_id] for image_id in image_ids]
-            else:
-                refs = self.data["refs"]
-            if not len(cat_ids) == 0:
-                refs = [ref for ref in refs if ref["category_id"] in cat_ids]
-            if not len(ref_ids) == 0:
-                refs = [ref for ref in refs if ref["ref_id"] in ref_ids]
-            if not len(split) == 0:
-                if split in ["testA", "testB", "testC"]:
-                    refs = [
-                        ref for ref in refs if split[-1] in ref["split"]
-                    ]  # we also consider testAB, testBC, ...
-                elif split in ["testAB", "testBC", "testAC"]:
-                    refs = [
-                        ref for ref in refs if ref["split"] == split
-                    ]  # rarely used I guess...
-                elif split == "test":
-                    refs = [ref for ref in refs if "test" in ref["split"]]
-                elif split == "train" or split == "val":
-                    refs = [ref for ref in refs if ref["split"] == split]
-                else:
-                    print("No such split [%s]" % split)
-                    sys.exit()
-        ref_ids = [ref["ref_id"] for ref in refs]
-        return ref_ids
-
-    def getAnnIds(self, image_ids=[], cat_ids=[], ref_ids=[]):
-        image_ids = image_ids if type(image_ids) == list else [image_ids]
-        cat_ids = cat_ids if type(cat_ids) == list else [cat_ids]
-        ref_ids = ref_ids if type(ref_ids) == list else [ref_ids]
-
-        if len(image_ids) == len(cat_ids) == len(ref_ids) == 0:
-            ann_ids = [ann["id"] for ann in self.data["annotations"]]
-        else:
-            if not len(image_ids) == 0:
-                lists = [
-                    self.imgToAnns[image_id]
-                    for image_id in image_ids
-                    if image_id in self.imgToAnns
-                ]  # list of [anns]
-                anns = list(itertools.chain.from_iterable(lists))
-            else:
-                anns = self.data["annotations"]
-            if not len(cat_ids) == 0:
-                anns = [ann for ann in anns if ann["category_id"] in cat_ids]
-            ann_ids = [ann["id"] for ann in anns]
-            if not len(ref_ids) == 0:
-                ids = set(ann_ids).intersection(
-                    set([self.Refs[ref_id]["ann_id"] for ref_id in ref_ids])
-                )
-        return ann_ids
-
-    def getImgIds(self, ref_ids=[]):
-        ref_ids = ref_ids if type(ref_ids) == list else [ref_ids]
-
-        if not len(ref_ids) == 0:
-            image_ids = list(set([self.Refs[ref_id]["image_id"] for ref_id in ref_ids]))
-        else:
-            image_ids = self.Imgs.keys()
-        return image_ids
-
-    def getCatIds(self):
-        return self.Cats.keys()
-
-    def loadRefs(self, ref_ids=[]):
-        if type(ref_ids) == list:
-            return [self.Refs[ref_id] for ref_id in ref_ids]
-        elif type(ref_ids) == int:
-            return [self.Refs[ref_ids]]
-
-    def loadAnns(self, ann_ids=[]):
-        if type(ann_ids) == list:
-            return [self.Anns[ann_id] for ann_id in ann_ids]
-        elif type(ann_ids) == int or type(ann_ids) == unicode:
-            return [self.Anns[ann_ids]]
-
-    def loadImgs(self, image_ids=[]):
-        if type(image_ids) == list:
-            return [self.Imgs[image_id] for image_id in image_ids]
-        elif type(image_ids) == int:
-            return [self.Imgs[image_ids]]
-
-    def loadCats(self, cat_ids=[]):
-        if type(cat_ids) == list:
-            return [self.Cats[cat_id] for cat_id in cat_ids]
-        elif type(cat_ids) == int:
-            return [self.Cats[cat_ids]]
-
-    def getRefBox(self, ref_id):
-        ref = self.Refs[ref_id]
-        ann = self.refToAnn[ref_id]
-        return ann["bbox"]  # [x, y, w, h]
-
-    def showRef(self, ref, seg_box="seg"):
-        ax = plt.gca()
-        # show image
-        image = self.Imgs[ref["image_id"]]
-        I = io.imread(osp.join(self.IMAGE_DIR, image["file_name"]))
-        ax.imshow(I)
-        # show refer expression
-        for sid, sent in enumerate(ref["sentences"]):
-            print("%s. %s" % (sid + 1, sent["sent"]))
-        # show segmentations
-        if seg_box == "seg":
-            ann_id = ref["ann_id"]
-            ann = self.Anns[ann_id]
-            polygons = []
-            color = []
-            c = "none"
-            if type(ann["segmentation"][0]) == list:
-                # polygon used for refcoco*
-                for seg in ann["segmentation"]:
-                    poly = np.array(seg).reshape((len(seg) / 2, 2))
-                    polygons.append(Polygon(poly, True, alpha=0.4))
-                    color.append(c)
-                p = PatchCollection(
-                    polygons,
-                    facecolors=color,
-                    edgecolors=(1, 1, 0, 0),
-                    linewidths=3,
-                    alpha=1,
-                )
-                ax.add_collection(p)  # thick yellow polygon
-                p = PatchCollection(
-                    polygons,
-                    facecolors=color,
-                    edgecolors=(1, 0, 0, 0),
-                    linewidths=1,
-                    alpha=1,
-                )
-                ax.add_collection(p)  # thin red polygon
-            else:
-                # mask used for refclef
-                rle = ann["segmentation"]
-                m = mask.decode(rle)
-                img = np.ones((m.shape[0], m.shape[1], 3))
-                color_mask = np.array([2.0, 166.0, 101.0]) / 255
-                for i in range(3):
-                    img[:, :, i] = color_mask[i]
-                ax.imshow(np.dstack((img, m * 0.5)))
-        # show bounding-box
-        elif seg_box == "box":
-            ann_id = ref["ann_id"]
-            ann = self.Anns[ann_id]
-            bbox = self.getRefBox(ref["ref_id"])
-            box_plot = Rectangle(
-                (bbox[0], bbox[1]),
-                bbox[2],
-                bbox[3],
-                fill=False,
-                edgecolor="green",
-                linewidth=3,
-            )
-            ax.add_patch(box_plot)
-
-    def getMask(self, ref):
-        # return mask, area and mask-center
-        ann = self.refToAnn[ref["ref_id"]]
-        image = self.Imgs[ref["image_id"]]
-        if type(ann["segmentation"][0]) == list:  # polygon
-            rle = mask.frPyObjects(ann["segmentation"], image["height"], image["width"])
-        else:
-            rle = ann["segmentation"]
-        m = mask.decode(rle)
-        m = np.sum(
-            m, axis=2
-        )  # sometimes there are multiple binary map (corresponding to multiple segs)
-        m = m.astype(np.uint8)  # convert to np.uint8
-        # compute area
-        area = sum(mask.area(rle))  # should be close to ann['area']
-        return {"mask": m, "area": area}
-        # # position
-        # position_x = np.mean(np.where(m==1)[1]) # [1] means columns (matlab style) -> x (c style)
-        # position_y = np.mean(np.where(m==1)[0]) # [0] means rows (matlab style)    -> y (c style)
-        # # mass position (if there were multiple regions, we use the largest one.)
-        # label_m = label(m, connectivity=m.ndim)
-        # regions = regionprops(label_m)
-        # if len(regions) > 0:
-        # 	largest_id = np.argmax(np.array([props.filled_area for props in regions]))
-        # 	largest_props = regions[largest_id]
-        # 	mass_y, mass_x = largest_props.centroid
-        # else:
-        # 	mass_x, mass_y = position_x, position_y
-        # # if centroid is not in mask, we find the closest point to it from mask
-        # if m[mass_y, mass_x] != 1:
-        # 	print('Finding closes mask point ...')
-        # 	kernel = np.ones((10, 10),np.uint8)
-        # 	me = cv2.erode(m, kernel, iterations = 1)
-        # 	points = zip(np.where(me == 1)[0].tolist(), np.where(me == 1)[1].tolist())  # row, col style
-        # 	points = np.array(points)
-        # 	dist   = np.sum((points - (mass_y, mass_x))**2, axis=1)
-        # 	id     = np.argsort(dist)[0]
-        # 	mass_y, mass_x = points[id]
-        # 	# return
-        # return {'mask': m, 'area': area, 'position_x': position_x, 'position_y': position_y, 'mass_x': mass_x, 'mass_y': mass_y}
-        # # show image and mask
-        # I = io.imread(osp.join(self.IMAGE_DIR, image['file_name']))
-        # plt.figure()
-        # plt.imshow(I)
-        # ax = plt.gca()
-        # img = np.ones( (m.shape[0], m.shape[1], 3) )
-        # color_mask = np.array([2.0,166.0,101.0])/255
-        # for i in range(3):
-        #     img[:,:,i] = color_mask[i]
-        # ax.imshow(np.dstack( (img, m*0.5) ))
-        # plt.show()
-
-    def showMask(self, ref):
-        M = self.getMask(ref)
-        msk = M["mask"]
-        ax = plt.gca()
-        ax.imshow(msk)
-
-
-if __name__ == "__main__":
-    refer = REFER(dataset="refcocog", splitBy="google")
-    ref_ids = refer.getRefIds()
-    print(len(ref_ids))
-
-    print(len(refer.Imgs))
-    print(len(refer.imgToRefs))
-
-    ref_ids = refer.getRefIds(split="train")
-    print("There are %s training referred objects." % len(ref_ids))
-
-    for ref_id in ref_ids:
-        ref = refer.loadRefs(ref_id)[0]
-        if len(ref["sentences"]) < 2:
-            continue
-
-        pprint(ref)
-        print("The label is %s." % refer.Cats[ref["category_id"]])
-        plt.figure()
-        refer.showRef(ref, seg_box="box")
-        plt.show()
-
-        # plt.figure()
-        # refer.showMask(ref)
-        # plt.show()

lisa_on_cuda/utils/refer_seg_dataset.py

@@ -1,277 +0,0 @@
-import os
-import random
-
-import cv2
-import numpy as np
-import torch
-import torch.nn.functional as F
-from pycocotools import mask
-from transformers import CLIPImageProcessor
-
-from lisa_on_cuda.llava import conversation as conversation_lib
-from lisa_on_cuda.segment_anything.utils.transforms import ResizeLongestSide
-
-from .grefer import G_REFER
-from .refer import REFER
-from .utils import ANSWER_LIST, SHORT_QUESTION_LIST
-
-
-class ReferSegDataset(torch.utils.data.Dataset):
-    pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
-    pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
-    img_size = 1024
-    ignore_label = 255
-
-    def __init__(
-        self,
-        base_image_dir,
-        tokenizer,
-        vision_tower,
-        samples_per_epoch=500 * 8 * 2 * 10,
-        precision: str = "fp32",
-        image_size: int = 224,
-        num_classes_per_sample: int = 3,
-        exclude_val=False,
-        refer_seg_data="refclef||refcoco||refcoco+||refcocog",
-    ):
-        self.exclude_val = exclude_val
-        self.samples_per_epoch = samples_per_epoch
-        self.num_classes_per_sample = num_classes_per_sample
-
-        self.base_image_dir = base_image_dir
-        self.image_size = image_size
-        self.tokenizer = tokenizer
-        self.precision = precision
-        self.transform = ResizeLongestSide(image_size)
-        self.clip_image_processor = CLIPImageProcessor.from_pretrained(vision_tower)
-
-        self.short_question_list = SHORT_QUESTION_LIST
-        self.answer_list = ANSWER_LIST
-
-        DATA_DIR = os.path.join(base_image_dir, "refer_seg")
-        self.refer_seg_ds_list = refer_seg_data.split(
-            "||"
-        )  # ['refclef', 'refcoco', 'refcoco+', 'refcocog']
-        self.refer_seg_data = {}
-        for ds in self.refer_seg_ds_list:
-            if ds == "refcocog":
-                splitBy = "umd"
-            else:
-                splitBy = "unc"
-
-            if ds == "grefcoco":
-                refer_api = G_REFER(DATA_DIR, ds, splitBy)
-            else:
-                refer_api = REFER(DATA_DIR, ds, splitBy)
-            ref_ids_train = refer_api.getRefIds(split="train")
-            images_ids_train = refer_api.getImgIds(ref_ids=ref_ids_train)
-            refs_train = refer_api.loadRefs(ref_ids=ref_ids_train)
-
-            refer_seg_ds = {}
-            refer_seg_ds["images"] = []
-            loaded_images = refer_api.loadImgs(image_ids=images_ids_train)
-
-            for item in loaded_images:
-                item = item.copy()
-                if ds == "refclef":
-                    item["file_name"] = os.path.join(
-                        DATA_DIR, "images/saiapr_tc-12", item["file_name"]
-                    )
-                else:
-                    item["file_name"] = os.path.join(
-                        DATA_DIR, "images/mscoco/images/train2014", item["file_name"]
-                    )
-                refer_seg_ds["images"].append(item)
-            refer_seg_ds["annotations"] = refer_api.Anns  # anns_train
-
-            print(
-                "dataset {} (refs {}) (train split) has {} images and {} annotations.".format(
-                    ds,
-                    splitBy,
-                    len(refer_seg_ds["images"]),
-                    len(refer_seg_ds["annotations"]),
-                )
-            )
-
-            img2refs = {}
-            for ref in refs_train:
-                image_id = ref["image_id"]
-                img2refs[image_id] = img2refs.get(image_id, []) + [
-                    ref,
-                ]
-            refer_seg_ds["img2refs"] = img2refs
-            self.refer_seg_data[ds] = refer_seg_ds
-
-    def __len__(self):
-        return self.samples_per_epoch
-
-    def preprocess(self, x: torch.Tensor) -> torch.Tensor:
-        """Normalize pixel values and pad to a square input."""
-        # Normalize colors
-        x = (x - self.pixel_mean) / self.pixel_std
-
-        # Pad
-        h, w = x.shape[-2:]
-        padh = self.img_size - h
-        padw = self.img_size - w
-        x = F.pad(x, (0, padw, 0, padh))
-        return x
-
-    def __getitem__(self, idx):
-        ds = random.randint(0, len(self.refer_seg_ds_list) - 1)
-        ds = self.refer_seg_ds_list[ds]
-        refer_seg_ds = self.refer_seg_data[ds]
-        images = refer_seg_ds["images"]
-        annotations = refer_seg_ds["annotations"]
-        img2refs = refer_seg_ds["img2refs"]
-        idx = random.randint(0, len(images) - 1)
-        image_info = images[idx]
-        image_path = image_info["file_name"]
-        image_id = image_info["id"]
-        refs = img2refs[image_id]
-        if len(refs) == 0:
-            return self.__getitem__(0)
-
-        sents = []
-        ann_ids = []
-        for ref in refs:
-            for sent in ref["sentences"]:
-                text = sent["sent"]
-                sents.append(text)
-                ann_ids.append(ref["ann_id"])
-        if len(sents) >= self.num_classes_per_sample:
-            sampled_inds = np.random.choice(
-                list(range(len(sents))), size=self.num_classes_per_sample, replace=False
-            )
-        else:
-            sampled_inds = list(range(len(sents)))
-        sampled_sents = np.vectorize(sents.__getitem__)(sampled_inds).tolist()
-        # sampled_ann_ids = np.vectorize(ann_ids.__getitem__)(sampled_inds).tolist()
-        sampled_ann_ids = [ann_ids[ind] for ind in sampled_inds]
-        sampled_classes = sampled_sents
-        image = cv2.imread(image_path)
-        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-
-        # preprocess image for clip
-        image_clip = self.clip_image_processor.preprocess(image, return_tensors="pt")[
-            "pixel_values"
-        ][0]
-
-        image = self.transform.apply_image(image)  # preprocess image for sam
-        resize = image.shape[:2]
-
-        questions = []
-        answers = []
-        for text in sampled_classes:
-            text = text.strip()
-            assert len(text.split("||")) == 1
-            question_template = random.choice(self.short_question_list)
-            questions.append(question_template.format(class_name=text.lower()))
-            answers.append(random.choice(self.answer_list))
-
-        conversations = []
-        conv = conversation_lib.default_conversation.copy()
-
-        i = 0
-        while i < len(questions):
-            conv.messages = []
-            conv.append_message(conv.roles[0], questions[i])
-            conv.append_message(conv.roles[1], answers[i])
-            conversations.append(conv.get_prompt())
-            i += 1
-
-        image = self.preprocess(torch.from_numpy(image).permute(2, 0, 1).contiguous())
-
-        flag = False
-        masks = []
-        for ann_id in sampled_ann_ids:
-            if isinstance(ann_id, list):
-                flag = True
-                if -1 in ann_id:
-                    assert len(ann_id) == 1
-                    m = np.zeros((image_info["height"], image_info["width"])).astype(
-                        np.uint8
-                    )
-                else:
-                    m_final = np.zeros(
-                        (image_info["height"], image_info["width"])
-                    ).astype(np.uint8)
-                    for ann_id_i in ann_id:
-                        ann = annotations[ann_id_i]
-
-                        if len(ann["segmentation"]) == 0:
-                            m = np.zeros(
-                                (image_info["height"], image_info["width"])
-                            ).astype(np.uint8)
-                        else:
-                            if type(ann["segmentation"][0]) == list:  # polygon
-                                rle = mask.frPyObjects(
-                                    ann["segmentation"],
-                                    image_info["height"],
-                                    image_info["width"],
-                                )
-                            else:
-                                rle = ann["segmentation"]
-                                for i in range(len(rle)):
-                                    if not isinstance(rle[i]["counts"], bytes):
-                                        rle[i]["counts"] = rle[i]["counts"].encode()
-                            m = mask.decode(rle)
-                            m = np.sum(
-                                m, axis=2
-                            )  # sometimes there are multiple binary map (corresponding to multiple segs)
-                            m = m.astype(np.uint8)  # convert to np.uint8
-                        m_final = m_final | m
-                    m = m_final
-                masks.append(m)
-                continue
-
-            ann = annotations[ann_id]
-
-            if len(ann["segmentation"]) == 0:
-                m = np.zeros((image_info["height"], image_info["width"])).astype(
-                    np.uint8
-                )
-                masks.append(m)
-                continue
-
-            if type(ann["segmentation"][0]) == list:  # polygon
-                rle = mask.frPyObjects(
-                    ann["segmentation"], image_info["height"], image_info["width"]
-                )
-            else:
-                rle = ann["segmentation"]
-                for i in range(len(rle)):
-                    if not isinstance(rle[i]["counts"], bytes):
-                        rle[i]["counts"] = rle[i]["counts"].encode()
-            m = mask.decode(rle)
-            m = np.sum(
-                m, axis=2
-            )  # sometimes there are multiple binary map (corresponding to multiple segs)
-            m = m.astype(np.uint8)  # convert to np.uint8
-            masks.append(m)
-
-        masks = np.stack(masks, axis=0)
-
-        # if ds == 'grefcoco' and flag:
-        #     import shutil
-        #     image_name = image_path.split("/")[-1]
-        #     save_dir = os.path.join("/group/30042/xlai/LISA_refactor_final/debug", image_name.split(".")[0])
-        #     os.makedirs(save_dir, exist_ok=True)
-        #     shutil.copy(image_path, save_dir)
-        #     for i in range(masks.shape[0]):
-        #         cv2.imwrite(os.path.join(save_dir, "{}_{}_{}.jpg".format(image_name, i, sampled_classes[i])), masks[i].astype(np.int32) * 100)
-
-        masks = torch.from_numpy(masks)
-        label = torch.ones(masks.shape[1], masks.shape[2]) * self.ignore_label
-
-        return (
-            image_path,
-            image,
-            image_clip,
-            conversations,
-            masks,
-            label,
-            resize,
-            questions,
-            sampled_classes,
-        )

lisa_on_cuda/utils/sem_seg_dataset.py

@@ -1,335 +0,0 @@
-import glob
-import json
-import os
-import random
-
-import cv2
-import numpy as np
-import torch
-import torch.nn.functional as F
-from PIL import Image
-from pycocotools.coco import COCO
-from transformers import CLIPImageProcessor
-
-from lisa_on_cuda.llava import conversation as conversation_lib
-from lisa_on_cuda.segment_anything.utils.transforms import ResizeLongestSide
-
-from .utils import ANSWER_LIST, SHORT_QUESTION_LIST
-
-
-def init_mapillary(base_image_dir):
-    mapillary_data_root = os.path.join(base_image_dir, "mapillary")
-    with open(os.path.join(mapillary_data_root, "config_v2.0.json")) as f:
-        mapillary_classes = json.load(f)["labels"]
-    mapillary_classes = [x["readable"].lower() for x in mapillary_classes]
-    mapillary_classes = np.array(mapillary_classes)
-    mapillary_labels = sorted(
-        glob.glob(
-            os.path.join(mapillary_data_root, "training", "v2.0", "labels", "*.png")
-        )
-    )
-    mapillary_images = [
-        x.replace(".png", ".jpg").replace("v2.0/labels", "images")
-        for x in mapillary_labels
-    ]
-    print("mapillary: ", len(mapillary_images))
-    return mapillary_classes, mapillary_images, mapillary_labels
-
-
-def init_ade20k(base_image_dir):
-    with open("utils/ade20k_classes.json", "r") as f:
-        ade20k_classes = json.load(f)
-    ade20k_classes = np.array(ade20k_classes)
-    image_ids = sorted(
-        os.listdir(os.path.join(base_image_dir, "ade20k/images", "training"))
-    )
-    ade20k_image_ids = []
-    for x in image_ids:
-        if x.endswith(".jpg"):
-            ade20k_image_ids.append(x[:-4])
-    ade20k_images = []
-    for image_id in ade20k_image_ids:  # self.descriptions:
-        ade20k_images.append(
-            os.path.join(
-                base_image_dir,
-                "ade20k",
-                "images",
-                "training",
-                "{}.jpg".format(image_id),
-            )
-        )
-    ade20k_labels = [
-        x.replace(".jpg", ".png").replace("images", "annotations")
-        for x in ade20k_images
-    ]
-    print("ade20k: ", len(ade20k_images))
-    return ade20k_classes, ade20k_images, ade20k_labels
-
-
-def init_cocostuff(base_image_dir):
-    cocostuff_classes = []
-    with open("utils/cocostuff_classes.txt") as f:
-        for line in f.readlines()[1:]:
-            cocostuff_classes.append(line.strip().split(": ")[-1])
-    cocostuff_classes = np.array(cocostuff_classes)
-    cocostuff_images = []
-
-    cocostuff_labels = glob.glob(
-        os.path.join(base_image_dir, "cocostuff", "train2017", "*.png")
-    )
-    cocostuff_images = [
-        x.replace(".png", ".jpg").replace("cocostuff", "coco") for x in cocostuff_labels
-    ]
-
-    print("cocostuff: ", len(cocostuff_images))
-    return cocostuff_classes, cocostuff_images, cocostuff_labels
-
-
-def init_paco_lvis(base_image_dir):
-    coco_api_paco_lvis = COCO(
-        os.path.join(
-            base_image_dir, "vlpart", "paco", "annotations", "paco_lvis_v1_train.json"
-        )
-    )
-    all_classes = coco_api_paco_lvis.loadCats(coco_api_paco_lvis.getCatIds())
-    class_map_paco_lvis = {}
-    for cat in all_classes:
-        cat_split = cat["name"].strip().split(":")
-        if len(cat_split) == 1:
-            name = cat_split[0].split("_(")[0]
-        else:
-            assert len(cat_split) == 2
-            obj, part = cat_split
-            obj = obj.split("_(")[0]
-            part = part.split("_(")[0]
-            name = (obj, part)
-        class_map_paco_lvis[cat["id"]] = name
-    img_ids = coco_api_paco_lvis.getImgIds()
-    print("paco_lvis: ", len(img_ids))
-    return class_map_paco_lvis, img_ids, coco_api_paco_lvis
-
-
-def init_pascal_part(base_image_dir):
-    coco_api_pascal_part = COCO(
-        os.path.join(base_image_dir, "vlpart", "pascal_part", "train.json")
-    )
-    all_classes = coco_api_pascal_part.loadCats(coco_api_pascal_part.getCatIds())
-    class_map_pascal_part = {}
-    for cat in all_classes:
-        cat_main, cat_part = cat["name"].strip().split(":")
-        name = (cat_main, cat_part)
-        class_map_pascal_part[cat["id"]] = name
-    img_ids = coco_api_pascal_part.getImgIds()
-    print("pascal_part: ", len(img_ids))
-    return class_map_pascal_part, img_ids, coco_api_pascal_part
-
-
-class SemSegDataset(torch.utils.data.Dataset):
-    pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
-    pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
-    img_size = 1024
-    ignore_label = 255
-
-    def __init__(
-        self,
-        base_image_dir,
-        tokenizer,
-        vision_tower,
-        samples_per_epoch=500 * 8 * 2 * 10,
-        precision: str = "fp32",
-        image_size: int = 224,
-        num_classes_per_sample: int = 3,
-        exclude_val=False,
-        sem_seg_data="ade20k||cocostuff||partimagenet||pascal_part||paco_lvis||mapillary",
-    ):
-        self.exclude_val = exclude_val
-        self.samples_per_epoch = samples_per_epoch
-        self.num_classes_per_sample = num_classes_per_sample
-
-        self.base_image_dir = base_image_dir
-        self.image_size = image_size
-        self.tokenizer = tokenizer
-        self.precision = precision
-        self.transform = ResizeLongestSide(image_size)
-        self.clip_image_processor = CLIPImageProcessor.from_pretrained(vision_tower)
-
-        self.short_question_list = SHORT_QUESTION_LIST
-        self.answer_list = ANSWER_LIST
-
-        self.data2list = {}
-        self.data2classes = {}
-
-        self.sem_seg_datas = sem_seg_data.split("||")
-        for ds in self.sem_seg_datas:
-            classes, images, labels = eval("init_{}".format(ds))(base_image_dir)
-            self.data2list[ds] = (images, labels)
-            self.data2classes[ds] = classes
-
-        if "cocostuff" in self.sem_seg_datas:
-            self.cocostuff_class2index = {
-                c: i for i, c in enumerate(self.data2classes["cocostuff"])
-            }
-
-    def __len__(self):
-        return self.samples_per_epoch
-
-    def preprocess(self, x: torch.Tensor) -> torch.Tensor:
-        """Normalize pixel values and pad to a square input."""
-        # Normalize colors
-        x = (x - self.pixel_mean) / self.pixel_std
-
-        # Pad
-        h, w = x.shape[-2:]
-        padh = self.img_size - h
-        padw = self.img_size - w
-        x = F.pad(x, (0, padw, 0, padh))
-        return x
-
-    def __getitem__(self, idx):
-        ds = random.randint(0, len(self.sem_seg_datas) - 1)
-        ds = self.sem_seg_datas[ds]
-
-        if ds in ["paco_lvis", "pascal_part"]:
-            class_map = self.data2classes[ds]
-            img_ids, coco_api = self.data2list[ds]
-            idx = random.randint(0, len(img_ids) - 1)
-            img_id = img_ids[idx]
-            image_info = coco_api.loadImgs([img_id])[0]
-            file_name = image_info["file_name"]
-            if ds == "pascal_part":
-                file_name = os.path.join(
-                    "VOCdevkit", "VOC2010", "JPEGImages", file_name
-                )
-                image_path = os.path.join(self.base_image_dir, "vlpart", ds, file_name)
-            elif ds == "paco_lvis":
-                image_path = os.path.join(self.base_image_dir, "coco", file_name)
-            image = cv2.imread(image_path)
-            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-
-            # preprocess image for clip
-            image_clip = self.clip_image_processor.preprocess(
-                image, return_tensors="pt"
-            )["pixel_values"][0]
-            image = self.transform.apply_image(image)  # preprocess image for sam
-            resize = image.shape[:2]
-            annIds = coco_api.getAnnIds(imgIds=image_info["id"])
-            anns = coco_api.loadAnns(annIds)
-            if len(anns) == 0:
-                return self.__getitem__(0)
-            if len(anns) >= self.num_classes_per_sample:
-                sampled_anns = np.random.choice(
-                    anns, size=self.num_classes_per_sample, replace=False
-                ).tolist()
-            else:
-                sampled_anns = anns
-            sampled_classes = []
-            for ann in sampled_anns:
-                sampled_cls = class_map[ann["category_id"]]
-                if isinstance(sampled_cls, tuple):
-                    obj, part = sampled_cls
-                    if random.random() < 0.5:
-                        name = obj + " " + part
-                    else:
-                        name = "the {} of the {}".format(part, obj)
-                else:
-                    name = sampled_cls
-                sampled_classes.append(name)
-
-        elif ds in ["ade20k", "cocostuff", "mapillary"]:
-            image, labels = self.data2list[ds]
-            idx = random.randint(0, len(image) - 1)
-            image_path = image[idx]
-            label_path = labels[idx]
-            label = Image.open(label_path)
-            label = np.array(label)
-            if ds == "ade20k":
-                label[label == 0] = 255
-                label -= 1
-                label[label == 254] = 255
-            elif ds == "cocostuff":
-                for c, i in self.cocostuff_class2index.items():
-                    if "-" in c:
-                        label[label == i] = 255
-            img = cv2.imread(image_path)
-            image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
-            # preprocess image for clip
-            image_clip = self.clip_image_processor.preprocess(
-                image, return_tensors="pt"
-            )["pixel_values"][0]
-            image = self.transform.apply_image(image)  # preprocess image for sam
-            resize = image.shape[:2]
-            unique_label = np.unique(label).tolist()
-            if 255 in unique_label:
-                unique_label.remove(255)
-            if len(unique_label) == 0:
-                return self.__getitem__(0)
-
-            classes = [self.data2classes[ds][class_id] for class_id in unique_label]
-            if len(classes) >= self.num_classes_per_sample:
-                sampled_classes = np.random.choice(
-                    classes, size=self.num_classes_per_sample, replace=False
-                ).tolist()
-            else:
-                sampled_classes = classes
-
-        questions = []
-        answers = []
-        class_ids = []
-        for sampled_cls in sampled_classes:
-            text = sampled_cls
-
-            assert len(text.split("||")) == 1
-            question_template = random.choice(self.short_question_list)
-            questions.append(question_template.format(class_name=text.lower()))
-
-            answers.append(random.choice(self.answer_list))
-
-            if ds in ["paco_lvis", "pascal_part"]:
-                continue
-
-            class_id = self.data2classes[ds].tolist().index(sampled_cls)
-            class_ids.append(class_id)
-
-        conversations = []
-        conv = conversation_lib.default_conversation.copy()
-
-        i = 0
-        while i < len(questions):
-            conv.messages = []
-            conv.append_message(conv.roles[0], questions[i])
-            conv.append_message(conv.roles[1], answers[i])
-            conversations.append(conv.get_prompt())
-            i += 1
-
-        image = self.preprocess(torch.from_numpy(image).permute(2, 0, 1).contiguous())
-
-        if ds in ["paco_lvis", "pascal_part"]:
-            masks = []
-            for ann in sampled_anns:
-                try:
-                    masks.append(coco_api.annToMask(ann))
-                except Exception as e:
-                    print(e)
-                    return self.__getitem__(0)
-
-            masks = np.stack(masks, axis=0)
-            masks = torch.from_numpy(masks)
-            label = torch.ones(masks.shape[1], masks.shape[2]) * self.ignore_label
-
-        else:
-            label = torch.from_numpy(label).long()
-            masks = []
-            for class_id in class_ids:
-                masks.append(label == class_id)
-            masks = torch.stack(masks, dim=0)
-        return (
-            image_path,
-            image,
-            image_clip,
-            conversations,
-            masks,
-            label,
-            resize,
-            questions,
-            sampled_classes,
-        )

lisa_on_cuda/utils/session_logger.py

@@ -1,36 +0,0 @@
-import contextvars
-import logging
-from functools import wraps
-from typing import Callable
-
-logging_uuid = contextvars.ContextVar("uuid")
-formatter = '%(asctime)s | %(uuid)s [%(pathname)s:%(module)s %(lineno)d] %(levelname)s | %(message)s'
-
-
-loggingType = logging.CRITICAL | logging.ERROR | logging.WARNING | logging.INFO | logging.DEBUG
-
-
-def change_logging(level_log: loggingType = logging.INFO) -> None:
-    old_factory = logging.getLogRecordFactory()
-
-    def record_factory(*args, **kwargs):
-        record = old_factory(*args, **kwargs)
-        record.uuid = logging_uuid.get("uuid")
-        if isinstance(record.msg, str):
-            record.msg = record.msg.replace("\\", "\\\\").replace("\n", "\\n")
-        return record
-
-    logging.setLogRecordFactory(record_factory)
-    logging.basicConfig(level=level_log, format=formatter, force=True)
-
-
-def set_uuid_logging(func: Callable) -> Callable:
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        import uuid
-
-        current_uuid = f"{uuid.uuid4()}"
-        logging_uuid.set(current_uuid)
-        return func(*args, **kwargs)
-
-    return wrapper

lisa_on_cuda/utils/vqa_dataset.py

@@ -1,135 +0,0 @@
-import json
-import os
-import random
-
-import cv2
-import torch
-import torch.nn.functional as F
-from transformers import CLIPImageProcessor
-
-from lisa_on_cuda.llava import conversation as conversation_lib
-from lisa_on_cuda.segment_anything.utils.transforms import ResizeLongestSide
-
-from .utils import DEFAULT_IMAGE_TOKEN
-
-
-def preprocess_multimodal(source, mm_use_im_start_end):
-    for sentence in source:
-        if DEFAULT_IMAGE_TOKEN in sentence["value"]:
-            sentence["value"] = (
-                sentence["value"].replace(DEFAULT_IMAGE_TOKEN, "").strip()
-            )
-            sentence["value"] = DEFAULT_IMAGE_TOKEN + "\n" + sentence["value"]
-            sentence["value"] = sentence["value"].strip()
-            if "mmtag" in conversation_lib.default_conversation.version:
-                sentence["value"] = sentence["value"].replace(
-                    DEFAULT_IMAGE_TOKEN, "<Image>" + DEFAULT_IMAGE_TOKEN + "</Image>"
-                )
-    return source
-
-
-class VQADataset(torch.utils.data.Dataset):
-    pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
-    pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
-    img_size = 1024
-    ignore_label = 255
-
-    def __init__(
-        self,
-        base_image_dir,
-        tokenizer,
-        vision_tower,
-        samples_per_epoch=500 * 8 * 2 * 10,
-        precision: str = "fp32",
-        image_size: int = 224,
-        num_classes_per_sample: int = 3,
-        exclude_val=False,
-        vqa_data="llava_instruct_150k",
-    ):
-        self.exclude_val = exclude_val
-        self.samples_per_epoch = samples_per_epoch
-        self.num_classes_per_sample = num_classes_per_sample
-
-        self.base_image_dir = base_image_dir
-        self.image_size = image_size
-        self.tokenizer = tokenizer
-        self.precision = precision
-        self.transform = ResizeLongestSide(image_size)
-        self.clip_image_processor = CLIPImageProcessor.from_pretrained(vision_tower)
-
-        DATA_DIR = os.path.join(base_image_dir, "llava_dataset")
-        self.vqa_image_root = os.path.join(base_image_dir, "coco/train2017")
-        with open(os.path.join(DATA_DIR, "{}.json".format(vqa_data))) as f:
-            vqa_data = json.load(f)
-        self.vqa_data = vqa_data
-
-        print("vqa_data: ", len(self.vqa_data))
-
-    def __len__(self):
-        return self.samples_per_epoch
-
-    def preprocess(self, x: torch.Tensor) -> torch.Tensor:
-        """Normalize pixel values and pad to a square input."""
-        # Normalize colors
-        x = (x - self.pixel_mean) / self.pixel_std
-
-        # Pad
-        h, w = x.shape[-2:]
-        padh = self.img_size - h
-        padw = self.img_size - w
-        x = F.pad(x, (0, padw, 0, padh))
-        return x
-
-    def __getitem__(self, idx):
-        idx = random.randint(0, len(self.vqa_data) - 1)
-        item = self.vqa_data[idx]
-        image_path = os.path.join(self.vqa_image_root, item["image"])
-        image = cv2.imread(image_path)
-        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-        ori_size = image.shape[:2]
-        image_clip = self.clip_image_processor.preprocess(image, return_tensors="pt")[
-            "pixel_values"
-        ][
-            0
-        ]  # preprocess image for clip
-
-        image = self.transform.apply_image(image)  # preprocess image for sam
-        resize = image.shape[:2]
-
-        conv = conversation_lib.default_conversation.copy()
-        source = item["conversations"]
-        source = preprocess_multimodal(
-            source,
-            mm_use_im_start_end=conv.sep_style == conversation_lib.SeparatorStyle.TWO,
-        )
-        roles = {"human": conv.roles[0], "gpt": conv.roles[1]}
-        conversations = []
-        if roles[source[0]["from"]] != conv.roles[0]:
-            # Skip the first one if it is not from human
-            source = source[1:]
-        conv.messages = []
-        for j, sentence in enumerate(source):
-            role = roles[sentence["from"]]
-            assert role == conv.roles[j % 2], f"{i}"
-            conv.append_message(role, sentence["value"])
-        conversations.append(conv.get_prompt())
-
-        questions = conversations
-        sampled_classes = conversations
-
-        image = self.preprocess(torch.from_numpy(image).permute(2, 0, 1).contiguous())
-
-        masks = torch.rand(0, *ori_size)
-        label = torch.ones(ori_size) * self.ignore_label
-
-        return (
-            image_path,
-            image,
-            image_clip,
-            conversations,
-            masks,
-            label,
-            resize,
-            questions,
-            sampled_classes,
-        )