Spaces:
Runtime error
Runtime error
File size: 7,563 Bytes
8fbc209 a7a2242 f16e094 8fbc209 062730b 8fbc209 f16e094 8fbc209 f16e094 8fbc209 f16e094 8fbc209 f16e094 9ac6d2f f16e094 062730b 2161a6c 4cc7c4e a7a2242 4cc7c4e b3a27e6 4cc7c4e b3a27e6 dd24d8c f16e094 062730b a7a2242 34a094f a7a2242 7c236a6 a7a2242 062730b f16e094 8fbc209 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 |
import gradio as gr
import spaces
import os
import time
from PIL import Image
from models.mllava import MLlavaProcessor, LlavaForConditionalGeneration, chat_mllava, MLlavaForConditionalGeneration
from typing import List
processor = MLlavaProcessor.from_pretrained("TIGER-Lab/Mantis-llava-7b-v1.1")
model = LlavaForConditionalGeneration.from_pretrained("TIGER-Lab/Mantis-llava-7b-v1.1")
@spaces.GPU
def generate(text:str, images:List[Image.Image], history: List[dict], **kwargs):
global processor, model
model = model.to("cuda")
if not images:
images = None
for text, history in chat_mllava(text, images, model, processor, history=history, stream=True, **kwargs):
yield text
return text
def enable_next_image(uploaded_images, image):
uploaded_images.append(image)
return uploaded_images, gr.MultimodalTextbox(value=None, interactive=False)
def add_message(history, message):
if message["files"]:
for file in message["files"]:
history.append([(file,), None])
if message["text"]:
history.append([message["text"], None])
return history, gr.MultimodalTextbox(value=None)
def print_like_dislike(x: gr.LikeData):
print(x.index, x.value, x.liked)
def get_chat_history(history):
chat_history = []
for i, message in enumerate(history):
if isinstance(message[0], str):
chat_history.append({"role": "user", "text": message[0]})
if i != len(history) - 1:
assert message[1], "The bot message is not provided, internal error"
chat_history.append({"role": "assistant", "text": message[1]})
else:
assert not message[1], "the bot message internal error, get: {}".format(message[1])
chat_history.append({"role": "assistant", "text": ""})
return chat_history
def get_chat_images(history):
images = []
for message in history:
if isinstance(message[0], tuple):
images.extend(message[0])
return images
def bot(history):
print(history)
cur_messages = {"text": "", "images": []}
for message in history[::-1]:
if message[1]:
break
if isinstance(message[0], str):
cur_messages["text"] = message[0] + " " + cur_messages["text"]
elif isinstance(message[0], tuple):
cur_messages["images"].extend(message[0])
cur_messages["text"] = cur_messages["text"].strip()
cur_messages["images"] = cur_messages["images"][::-1]
if not cur_messages["text"]:
raise gr.Error("Please enter a message")
if cur_messages['text'].count("<image>") < len(cur_messages['images']):
gr.Warning("The number of images uploaded is more than the number of <image> placeholders in the text. Will automatically prepend <image> to the text.")
cur_messages['text'] = "<image> "* (len(cur_messages['images']) - cur_messages['text'].count("<image>")) + cur_messages['text']
history[-1][0] = cur_messages["text"]
if cur_messages['text'].count("<image>") > len(cur_messages['images']):
gr.Warning("The number of images uploaded is less than the number of <image> placeholders in the text. Will automatically remove extra <image> placeholders from the text.")
cur_messages['text'] = cur_messages['text'][::-1].replace("<image>"[::-1], "", cur_messages['text'].count("<image>") - len(cur_messages['images']))[::-1]
history[-1][0] = cur_messages["text"]
chat_history = get_chat_history(history)
chat_images = get_chat_images(history)
generation_kwargs = {
"max_new_tokens": 4096,
"temperature": 0.2,
"top_p": 1.0,
"do_sample": True,
}
print(None, chat_images, chat_history, generation_kwargs)
response = generate(None, chat_images, chat_history, **generation_kwargs)
for _output in response:
history[-1][1] = _output
time.sleep(0.05)
yield history
def build_demo():
with gr.Blocks() as demo:
with gr.Row():
with gr.Column():
gr.Markdown(""" # Mantis
Mantis is a multimodal conversational AI model that can chat with users about images and text. It's optimized for multi-image reasoning, where inverleaved text and images can be used to generate responses.
| [Github](https://github.com/TIGER-AI-Lab/Mantis) | [Blog](https://tiger-ai-lab.github.io/Blog/mantis) | [Models](https://huggingface.co/collections/TIGER-Lab/mantis-6619b0834594c878cdb1d6e4) |
""")
# gr.Image("./barchart_single_image_vqa.jpeg")
with gr.Column():
gr.Image("./barchart.jpeg")
gr.Markdown("""## Chat with Mantis
Mantis supports interleaved text-image input format, where you can simply use the placeholder `<image>` to indicate the position of uploaded images.
The model is optimized for multi-image reasoning, while preserving the ability to chat about text and images in a single conversation.
(The model currently serving is [Mantis-bakllava-7b](https://huggingface.co/TIGER-Lab/Mantis-bakllava-7b))
""")
chatbot = gr.Chatbot(line_breaks=True)
chat_input = gr.MultimodalTextbox(interactive=True, file_types=["image"], placeholder="Enter message or upload images. Please use <image> to indicate the position of uploaded images", show_label=True)
chat_msg = chat_input.submit(add_message, [chatbot, chat_input], [chatbot, chat_input])
bot_msg = chat_msg.success(bot, chatbot, chatbot, api_name="bot_response")
chatbot.like(print_like_dislike, None, None)
with gr.Row():
send_button = gr.Button("Send")
clear_button = gr.ClearButton([chatbot, chat_input])
send_button.click(
add_message, [chatbot, chat_input], [chatbot, chat_input]
).then(
bot, chatbot, chatbot, api_name="bot_response"
)
gr.Examples(
examples=[
{
"text": "<image> <image> How many dices are there in image 1 and image 2 respectively?",
"files": ["./examples/image10.jpg", "./examples/image11.jpg"]
},
{
"text": "<image> <image> <image> Which image shows a different mood of character from the others?",
"files": ["./examples/image12.jpg", "./examples/image13.jpg", "./examples/image14.jpg"]
},
{
"text": "<image> <image> What's the difference between these two images? Please describe as much as you can.",
"files": ["./examples/image1.jpg", "./examples/image2.jpg"]
},
{
"text": "<image> <image> Which image shows an older dog?",
"files": ["./examples/image8.jpg", "./examples/image9.jpg"]
},
{
"text": "Write a description for the given image sequence in a single paragraph, what is happening in this episode?",
"files": ["./examples/image3.jpg", "./examples/image4.jpg", "./examples/image5.jpg", "./examples/image6.jpg", "./examples/image7.jpg"]
},
],
inputs=[chat_input],
)
return demo
if __name__ == "__main__":
demo = build_demo()
demo.launch() |