app.py

import string
import gradio as gr
import requests
import torch
from models.VLE import VLEForVQA, VLEProcessor, VLEForVQAPipeline
from PIL import Image

# model_name="hfl/vle-base-for-vqa"
# model = VLEForVQA.from_pretrained(model_name)
# vle_processor = VLEProcessor.from_pretrained(model_name)
# vqa_pipeline = VLEForVQAPipeline(model=model, device='cpu', vle_processor=vle_processor)


from transformers import BlipForQuestionAnswering, BlipProcessor

# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-capfilt-large")
# model_vqa = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-capfilt-large").to(device)

from transformers import BlipProcessor, BlipForConditionalGeneration

# cap_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
# cap_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")



def caption(input_image):
    inputs = cap_processor(input_image, return_tensors="pt")
    # inputs["num_beams"] = 1
    # inputs['num_return_sequences'] =1
    out = cap_model.generate(**inputs)
    return "\n".join(cap_processor.batch_decode(out, skip_special_tokens=True))
import openai
import os
openai.api_key= os.getenv('openai_appkey') 
def gpt3_short(question,vqa_answer,caption):
    vqa_answer,vqa_score=vqa_answer
    prompt="This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+"A: "+vqa_answer[0]+", socre:"+str(vqa_score[0])+\
           "; B: "+vqa_answer[1]+", score:"+str(vqa_score[1])+"; C: "+vqa_answer[2]+", score:"+str(vqa_score[2])+\
            "; D: "+vqa_answer[3]+', score:'+str(vqa_score[3])+\
           ". Choose A if it is not in conflict with the description of the picture and A's score is bigger than 0.8; otherwise choose the B, C or D based on the description."
    
    # prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer."
    response = openai.Completion.create(
    engine="text-davinci-003",
    prompt=prompt,
    max_tokens=10,
    n=1,
    stop=None,
    temperature=0.7,
    )
    answer = response.choices[0].text.strip()

    llm_ans=answer
    choice=set(["A","B","C","D"])
    llm_ans=llm_ans.replace("\n"," ").replace(":"," ").replace("."," " ).replace(","," ")
    sllm_ans=llm_ans.split(" ")
    for cho in sllm_ans:
      if cho in choice:
         llm_ans=cho
         break
    if llm_ans not in choice:
        llm_ans="A"
    llm_ans=vqa_answer[ord(llm_ans)-ord("A")]
    answer=llm_ans
    
    return answer
def gpt3_long(question,vqa_answer,caption):
    vqa_answer,vqa_score=vqa_answer
    # prompt="prompt: This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+"A: "+vqa_answer[0]+"socre:"+str(vqa_score[0])+\
    #        " B: "+vqa_answer[1]+" score:"+str(vqa_score[1])+" C: "+vqa_answer[2]+" score:"+str(vqa_score[2])+\
    #         " D: "+vqa_answer[3]+'score:'+str(vqa_score[3])+\
    #        "Tell me the right answer with a long sentence."
    prompt="This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+" "+vqa_answer[0]+", socre:"+str(vqa_score[0])+\
           ";   "+vqa_answer[1]+", score:"+str(vqa_score[1])+";  "+vqa_answer[2]+", score:"+str(vqa_score[2])+\
            ";  "+vqa_answer[3]+', score:'+str(vqa_score[3])+\
           ". Question: "+question+" Tell me the right answer with a sentence."
    # prompt="prompt: This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+" "+vqa_answer[0]+" socre:"+str(vqa_score[0])+\
    #        "   "+vqa_answer[1]+" score:"+str(vqa_score[1])+"  "+vqa_answer[2]+" score:"+str(vqa_score[2])+\
    #         "  "+vqa_answer[3]+'score:'+str(vqa_score[3])+\
    #        "Tell me the right answer with a long sentence."
    # prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer."
    response = openai.Completion.create(
    engine="text-davinci-003",
    prompt=prompt,
    max_tokens=30,
    n=1,
    stop=None,
    temperature=0.7,
    )
    answer = response.choices[0].text.strip()    
    return answer
def gpt3(question,vqa_answer,caption):
    prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer."
    response = openai.Completion.create(
    engine="text-davinci-003",
    prompt=prompt,
    max_tokens=30,
    n=1,
    stop=None,
    temperature=0.7,
    )
    answer = response.choices[0].text.strip()
    # return "input_text:\n"+prompt+"\n\n output_answer:\n"+answer
    return answer

def vle(input_image,input_text):
    vqa_answers = vqa_pipeline({"image":input_image, "question":input_text}, top_k=4)
    # return [" ".join([str(value) for key,value in vqa.items()] )for vqa in vqa_answers]
    return [vqa['answer'] for vqa in vqa_answers],[vqa['score'] for vqa in vqa_answers]
def inference_chat(input_image,input_text):
    cap=caption(input_image)
    print(cap)
    # inputs = processor(images=input_image, text=input_text,return_tensors="pt")
    # inputs["max_length"] = 10
    # inputs["num_beams"] = 5
    # inputs['num_return_sequences'] =4
    # out = model_vqa.generate(**inputs)
    # out=processor.batch_decode(out, skip_special_tokens=True)

    out=vle(input_image,input_text)
    # vqa="\n".join(out[0])
    # gpt3_out=gpt3(input_text,vqa,cap)
    gpt3_out=gpt3_long(input_text,out,cap)
    gpt3_out1=gpt3_short(input_text,out,cap)
    return out[0][0], gpt3_out,gpt3_out1
title = """# VQA with VLE and LLM"""
description = """We demonstrate three visual question answering systems built with VLE and LLM:



* VQA + LLM (short answer): The captioning model generates a caption of the image. We feed the caption, the question, and the answer candidates predicted by the VQA model to the LLM, and ask the LLM to select the most reasonable answer from the candidates.

* VQA + LLM (long answer): The pipeline is the same as VQA + LLM (short answer), except that the answer is freely generated by the LLM and not limited to VQA candidates.

For more details about VLE and the VQA pipeline, see [http://vle.hfl-rc.com](http://vle.hfl-rc.com)"""

with gr.Blocks(
    css="""
    .message.svelte-w6rprc.svelte-w6rprc.svelte-w6rprc {font-size: 20px; margin-top: 20px}
    #component-21 > div.wrap.svelte-w6rprc {height: 600px;}
    """
) as iface:
    state = gr.State([])
    #caption_output = None
    gr.Markdown(title)
    
    #gr.Markdown(article)

    with gr.Row():
        with gr.Column(scale=1):
            image_input = gr.Image(type="pil",label="VQA Image Input")
            with gr.Row():
                with gr.Column(scale=1):
                    chat_input = gr.Textbox(lines=1, label="VQA Question Input")
                    with gr.Row():
                        clear_button = gr.Button(value="Clear", interactive=True,width=30)
                        submit_button = gr.Button(
                            value="Submit", interactive=True, variant="primary"
                        )
                        '''
                    cap_submit_button = gr.Button(
                            value="Submit_CAP", interactive=True, variant="primary"
                        )
                    gpt3_submit_button = gr.Button(
                            value="Submit_GPT3", interactive=True, variant="primary"
                        )
                        '''
        with gr.Column():
            
            caption_output = gr.Textbox(lines=0, label="* VQA + LLM (short answer):\n The captioning model generates a caption \n of the image. We feed the caption")
            caption_output_v1 = gr.Textbox(lines=0, label="VQA + LLM (short answer)")
            gpt3_output_v1 = gr.Textbox(lines=0, label="VQA+LLM (long answer)")
            gr.Markdown(description)
            
            
        # image_input.change(
        #     lambda: ("", [],"","",""),
        #     [],
        #     [ caption_output, state,caption_output,gpt3_output_v1,caption_output_v1],
        #     queue=False,
        # )
        chat_input.submit(
                    inference_chat,
                    [
                        image_input,
                        chat_input,
                    ],
                    [ caption_output,gpt3_output_v1,caption_output_v1],
                )
        clear_button.click(
                        lambda: ("", [],"","",""),
                        [],
                        [chat_input,  state,caption_output,gpt3_output_v1,caption_output_v1],
                        queue=False,
                    )
        submit_button.click(
                        inference_chat,
                        [
                            image_input,
                            chat_input,
                        ],
                        [caption_output,gpt3_output_v1,caption_output_v1],
                    )
        '''
        cap_submit_button.click(
                        caption,
                        [
                            image_input,
                   
                        ],
                        [caption_output_v1],
                    )
        gpt3_submit_button.click(
                        gpt3,
                        [
                            chat_input,
                           caption_output ,
                            caption_output_v1,
                        ],
                        [gpt3_output_v1],
                    )
        '''
    examples=[['bird.jpeg',"How many birds are there in the tree?","2","2","2"],
              ['qa9.jpg',"What type of vehicle is being pulled by the horses ?",'carriage','sled','Sled'],
                ['upload4.jpg',"What is this old man doing?","fishing","fishing","Fishing"]]
    examples = gr.Examples(
       examples=examples,inputs=[image_input, chat_input,caption_output,caption_output_v1,gpt3_output_v1],
    )

iface.queue(concurrency_count=1, api_open=False, max_size=10)
iface.launch(enable_queue=True)