|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import sys |
|
from IPython.display import display, HTML |
|
from BLIP.models.blip import blip_decoder |
|
from google_drive_downloader import GoogleDriveDownloader as gdd |
|
from PIL import Image |
|
import requests |
|
import torch |
|
from torchvision import transforms |
|
from torchvision.transforms.functional import InterpolationMode |
|
from urllib.parse import urlparse |
|
from google_drive_downloader import GoogleDriveDownloader as gdd |
|
|
|
|
|
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') |
|
|
|
|
|
transform = transforms.Compose([ |
|
transforms.Resize((image_size,image_size),interpolation=InterpolationMode.BICUBIC), |
|
transforms.ToTensor(), |
|
transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)) |
|
]) |
|
|
|
model_url = "https://technionmail-my.sharepoint.com/personal/snoamr_campus_technion_ac_il/_layouts/15/download.aspx?share=EZxgXQaBXGREgDsQiaTcwAAB0z8jQA_hgAnwwPQDt8Dgew" |
|
model = blip_decoder(pretrained=model_url, image_size=384, vit='base') |
|
model.eval() |
|
model = model.to(device) |
|
|
|
def inference(raw_image): |
|
image = transform(raw_image).unsqueeze(0).to(device) |
|
with torch.no_grad(): |
|
caption = model.generate(image, sample=False, num_beams=1, max_length=60, min_length=5) |
|
return caption[0] |
|
|
|
|
|
inputs = [gr.Image(type='pil', interactive=False),] |
|
outputs = gr.outputs.Textbox(label="Caption") |
|
|
|
title = "FuseCap" |
|
description = "Gradio demo for FuseCap: Leveraging Large Language Models to Fuse Visual Data into Enriched Image Captions. This demo features a BLIP-based model, trained using FuseCap." |
|
|
|
article = "place holder" |
|
gr.Interface(inference, inputs, outputs, title=title, description=description, article=article, examples=[['birthday_dog.jpeg']]).launch(enable_queue=True) |
