tasks/text_ret.py

# --------------------------------------------------------
# X-Decoder -- Generalized Decoding for Pixel, Image, and Language
# Copyright (c) 2022 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Xueyan Zou (xueyan@cs.wisc.edu)
# --------------------------------------------------------

import torch
import numpy as np
from PIL import Image
from torchvision import transforms
from detectron2.data import MetadataCatalog
from xdecoder.language.loss import vl_similarity


t = []
t.append(transforms.Resize(224, interpolation=Image.BICUBIC))
transform_ret = transforms.Compose(t)
t = []
t.append(transforms.Resize(512, interpolation=Image.BICUBIC))
transform_grd = transforms.Compose(t)

metedata = MetadataCatalog.get('coco_2017_train_panoptic')

def text_retrieval(model, image, texts, inpainting_text, *args, **kwargs):
    out_str = ''
    with torch.no_grad():
        image = transform_ret(image)
        image = np.asarray(image)
        images = torch.from_numpy(image.copy()).permute(2,0,1).cuda()
        batch_inputs = [{'image': images, 'image_id': 0}]
        outputs = model.model.evaluate(batch_inputs)
        v_emb = torch.cat([x['captions'][-1:] for x in outputs])
        v_emb = v_emb / (v_emb.norm(dim=-1, keepdim=True) + 1e-7)
        
        texts = [x.strip() for x in texts.split(',')]
        model.model.sem_seg_head.predictor.lang_encoder.get_text_embeddings(texts, is_eval=False, name='caption', prompt=False)
        t_emb = getattr(model.model.sem_seg_head.predictor.lang_encoder, '{}_text_embeddings'.format('caption'))
        temperature = model.model.sem_seg_head.predictor.lang_encoder.logit_scale
        logits = vl_similarity(v_emb, t_emb, temperature)
        topk_prob, topk_idx = logits.softmax(-1)[0].topk(min(5, len(texts)))
        
        for prob, idx in zip(topk_prob, topk_idx):
            out_str += "{}:{:.2f}; ".format(texts[idx.item()], prob.item())
    torch.cuda.empty_cache()
    return None, out_str, None