init

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+tmp*

app.py

@@ -1,64 +1,229 @@
+# Copyright (2024) Bytedance Ltd. and/or its affiliates
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# copy and modify from: https://github.com/OpenGVLab/Ask-Anything/blob/main/video_chat2/demo/demo.py
+import spaces
+from copy import deepcopy
 import gradio as gr
-from huggingface_hub import InferenceClient
+from gradio.themes.utils import colors, fonts, sizes
+from tools.conversation import Chat, conv_templates
+from tools.utils import load_model_and_processor, file_to_base64
+from dataset.processor import Processor
+import os
+import torch
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
+device = 'cuda'
+model_path = os.getenv("MODEL_PATH", "/home/user/checkpoints/Tarsier-7b")
+max_n_frames = int(os.getenv("MAX_N_FRAMES", 8))
+debug = True
+
+# ========================================
+#             Model Initialization
+# ========================================
+def init_model():
+    print("Start Initialization...")
+    # if torch.cuda.is_available():
+    if not debug:
+        model, processor = load_model_and_processor(model_path, max_n_frames)
+    else:
+        print(f"No Valid GPU! Lauch in debug mode!")
+        processor = Processor(model_path, max_n_frames)
+        model = None
+    chat = Chat(model, processor, device, debug)
+    print('Initialization Finished')
+    return chat
+
+
+# ========================================
+#             Gradio Setting
+# ========================================
+def gradio_reset(chat_state, img_file, img_list):
+    if chat_state is not None:
+        chat_state.messages = []
+    img_file = None
+    if img_list is not None:
+        img_list = []
+    return None, gr.update(value=None, interactive=True), gr.update(value=None, interactive=True), gr.update(value=None, interactive=True), gr.update(placeholder='Please upload your video first', interactive=False),gr.update(value="Upload & Start Chat", interactive=True), chat_state, img_file, img_list
+
+
+def upload_img(gr_img, gr_video, gr_gif, chat_state, num_frames):
+    print(gr_img, gr_video)
+    conv_type = ''
+    if 'tarsier2-7b' in model_path.lower():
+        conv_type = 'tarsier2-7b'
+    elif '7b' in model_path.lower():
+        conv_type = 'tarsier-7b'
+    elif '13b' in model_path.lower():
+        conv_type = 'tarsier-13b'
+    elif '34b' in model_path.lower():
+        conv_type = 'tarsier-34b'
+    else:
+        raise ValueError(f"Unknow model: {model_path}")
+    chat_state = deepcopy(conv_templates[conv_type])
+
+    img_list = []
+    if gr_img is None and gr_video is None and gr_gif is None:
+        return None, None, None, gr.update(interactive=True), gr.update(interactive=True, placeholder='Please upload video/image first!'), chat_state, None, None
+    if gr_video or gr_img or gr_gif:
+        for img_file in [gr_video, gr_video, gr_gif]:
+            if img_file is not None:
+                break
+        return gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True, placeholder='Type and press Enter'), gr.update(value="Start Chatting", interactive=False), chat_state, img_file, img_list
+
+
+def gradio_ask(user_message, chatbot, chat_state):
+    if len(user_message) == 0:
+        return gr.update(interactive=True, placeholder='Input should not be empty!'), chatbot, chat_state
+    chat_state =  chat.ask(user_message, chat_state)
+    chatbot = chatbot + [[user_message, None]]
+    return '', chatbot, chat_state
+
+@spaces.GPU(duration=120)
+def gradio_answer(chatbot, chat_state, img_file, img_list, top_p, temperature, n_frames=None):
+    llm_message, chat_state, img_list = chat.answer(conv=chat_state, visual_data_file=img_file, images=img_list, n_frames=n_frames, max_new_tokens=512, num_beams=1, temperature=temperature, top_p=top_p)
+    chatbot[-1][1] = llm_message
+    print(chat_state)
+    print(f"Answer: {llm_message}")
+    return chatbot, chat_state, img_list
+
+
+class OpenGVLab(gr.themes.base.Base):
+    def __init__(
+        self,
+        *,
+        primary_hue=colors.blue,
+        secondary_hue=colors.sky,
+        neutral_hue=colors.gray,
+        spacing_size=sizes.spacing_md,
+        radius_size=sizes.radius_sm,
+        text_size=sizes.text_md,
+        font=(
+            fonts.GoogleFont("Noto Sans"),
+            "ui-sans-serif",
+            "sans-serif",
+        ),
+        font_mono=(
+            fonts.GoogleFont("IBM Plex Mono"),
+            "ui-monospace",
+            "monospace",
+        ),
     ):
-        token = message.choices[0].delta.content
+        super().__init__(
+            primary_hue=primary_hue,
+            secondary_hue=secondary_hue,
+            neutral_hue=neutral_hue,
+            spacing_size=spacing_size,
+            radius_size=radius_size,
+            text_size=text_size,
+            font=font,
+            font_mono=font_mono,
+        )
+        super().set(
+            body_background_fill="*neutral_50",
+        )
 
-        response += token
-        yield response
 
+gvlabtheme = OpenGVLab(primary_hue=colors.blue,
+        secondary_hue=colors.sky,
+        neutral_hue=colors.gray,
+        spacing_size=sizes.spacing_md,
+        radius_size=sizes.radius_sm,
+        text_size=sizes.text_md,
+        )
 
+logo_b64 = file_to_base64("assets/figures/tarsier_logo.jpg")
+title = f"""<center><a href="https://github.com/bytedance/tarsier"><img src="data:image/jpeg;base64,{logo_b64}" alt="Tarsier" border="0" style="margin: 0 auto; height: 140px;" /></a></center>"""
+description ="""<center><p><a href='https://github.com/bytedance/tarsier'><img src='https://img.shields.io/badge/Github-Code-blue'></a></p><p></center>
 """
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
 
 
-if __name__ == "__main__":
-    demo.launch()
+with gr.Blocks(title="Tarsier",theme=gvlabtheme,css="#chatbot {overflow:auto; height:500px;} #InputVideo {overflow:visible; height:320px;} footer {visibility: none}") as demo:
+    gr.Markdown(title)
+    gr.Markdown(description)
+    with gr.Row():
+        with gr.Column(scale=0.5, visible=True) as video_upload:
+            with gr.Column(elem_id="image", scale=0.5) as img_part:
+                with gr.Tab("Video", elem_id='video_tab'):
+                    up_video = gr.Video(interactive=True, include_audio=True, elem_id="video_upload", height=360)
+                with gr.Tab("Image", elem_id='image_tab'):
+                    up_image = gr.Image(type="filepath", interactive=True, elem_id="image_upload", height=360)
+                with gr.Tab("GIF", elem_id='gif_tab'):
+                    up_gif = gr.File(type="filepath", file_count="single", file_types=["gif"], interactive=True, elem_id="gif_upload", height=360)
+            upload_button = gr.Button(value="Upload & Start Chat", interactive=True, variant="primary")
+            clear = gr.Button("Restart")
+            
+            # num_beams = gr.Slider(
+            #     minimum=1,
+            #     maximum=10,
+            #     value=1,
+            #     step=1,
+            #     interactive=True,
+            #     label="beam search numbers)",
+            # )
+            
+            temperature = gr.Slider(
+                minimum=0.0,
+                maximum=1.0,
+                value=0.0,
+                step=0.1,
+                interactive=True,
+                label="Temperature",
+            )
+
+            top_p = gr.Slider(
+                minimum=0.1,
+                maximum=1.0,
+                value=1.0,
+                step=0.1,
+                interactive=True,
+                label="Top_p",
+            )
+            
+            num_frames = gr.Slider(
+                minimum=4,
+                maximum=16,
+                value=8,
+                step=2,
+                interactive=True,
+                label="#Frames",
+            )
+        
+        with gr.Column(visible=True)  as input_raws:
+            chat_state = gr.State()
+            img_list = gr.State()
+            img_file = gr.State()
+            chatbot = gr.Chatbot(elem_id="chatbot",label='VideoChat')
+            with gr.Row():
+                with gr.Column(scale=0.7):
+                    text_input = gr.Textbox(show_label=False, placeholder='Please upload your video first', interactive=False, container=False)
+                with gr.Column(scale=0.15, min_width=0):
+                    run = gr.Button("💭Send")
+                with gr.Column(scale=0.15, min_width=0):
+                    clear = gr.Button("🔄Clear️")     
+    
+    chat = init_model()
+    upload_button.click(upload_img, [up_image, up_video, up_gif, chat_state, num_frames], [up_image, up_video, up_gif, text_input, upload_button, chat_state, img_file, img_list])
+    
+    text_input.submit(gradio_ask, [text_input, chatbot, chat_state], [text_input, chatbot, chat_state]).then(
+        gradio_answer, [chatbot, chat_state, img_file, img_list, top_p, temperature, num_frames], [chatbot, chat_state, img_list]
+    )
+    run.click(gradio_ask, [text_input, chatbot, chat_state], [text_input, chatbot, chat_state]).then(
+        gradio_answer, [chatbot, chat_state, img_file, img_list, top_p, temperature, num_frames], [chatbot, chat_state, img_list]
+    )
+    run.click(lambda: "", None, text_input)  
+    clear.click(gradio_reset, [chat_state, img_file, img_list], [chatbot, up_image, up_video, up_gif, text_input, upload_button, chat_state, img_file, img_list], queue=False)
+
+
+demo.launch()
+# demo.launch(server_name="0.0.0.0", server_port=11451)

dataset/mm_dataset.py

@@ -0,0 +1,62 @@
+# Copyright (2024) Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from dataset.utils import get_visual_type, sample_frame_indices
+from .processor import Processor
+from tools.rw_utils import read_jsonlines
+
+class MMDataset(object):
+    def __init__(self, ann_path="", anns=None, processor:Processor=None):
+        self.processor = processor
+        if anns is None:
+            self.anns = []
+            if isinstance(ann_path, str):
+                ann_path = [ann_path]
+            for path in ann_path:
+                self.anns.extend(read_jsonlines(path))
+        else:
+            self.anns = anns
+
+    def __len__(self):
+        return len(self.anns)
+    
+    def __getitem__(self, index):
+        try:
+            ann = self.anns[index]
+
+            prompt = ann['text']['prompt']
+
+            video_file = ann['video_file']
+            visual_files = []
+            start_time = ann.get("start_time", 0)
+            end_time = ann.get("end_time", -1)
+            if isinstance(video_file, list):
+                # This is for MVBench/Episodic Reasoning
+                # The video_file are a list of sorted frames extract from the target video
+                for img_file in video_file:
+                    if get_visual_type(img_file) == 'image':
+                        visual_files.append(img_file)
+                frame_indices = sample_frame_indices(start_frame=0, total_frames=len(visual_files), n_frames=min(len(visual_files), self.processor.max_n_frames))
+                visual_files = [v for i,v in enumerate(visual_files) if i in frame_indices]    
+            else:
+                if get_visual_type(video_file) in ['image', 'video', 'gif']:
+                    visual_files.append(video_file)
+            assert len(visual_files) >= 0, f"Failed to load valid visual file from anns[{index}]!"
+            images = []
+            for v_f in visual_files:
+                images.extend(self.processor.load_images(v_f, start_time=start_time, end_time=end_time))
+            model_inputs = self.processor(prompt, images=images, edit_prompt=True, return_prompt=True)
+        except Exception as e:
+            print(f"Load data error: {e}")
+            return ann, None
+        return ann, model_inputs

dataset/processor.py

@@ -0,0 +1,160 @@
+# Copyright (2024) Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from PIL import Image
+from typing import List
+import torch
+from transformers import DataCollatorForSeq2Seq
+from transformers.models.llava import LlavaProcessor
+import re
+
+from .utils import sample_image, sample_video, sample_gif, get_visual_type
+
+ext2sampler = {
+    'image': sample_image,
+    'gif': sample_gif,
+    'video': sample_video
+}
+
+class CustomImageProcessor:
+    def __init__(self, processor) -> None:
+        self.processor = processor
+
+    def __call__(self, images: List[Image.Image], do_padding=False) -> torch.Tensor:
+        if do_padding:
+            images = [self.expand2square(
+                img,
+                tuple(int(x * 255) for x in self.processor.image_processor.image_mean)
+            ) for img in images]
+        else:
+            images = [self.resize2square(img) for img in images]
+        images_pixel = self.processor(text="", images=images, return_tensors="pt")['pixel_values']
+        return images_pixel  # [num_images, 3, 336, 336]
+
+    def expand2square(self, pil_img, background_color):
+        width, height = pil_img.size
+        if width == height:
+            return pil_img
+        elif width > height:
+            result = Image.new(pil_img.mode, (width, width), background_color)
+            result.paste(pil_img, (0, (width - height) // 2))
+            return result
+        else:
+            result = Image.new(pil_img.mode, (height, height), background_color)
+            result.paste(pil_img, ((height - width) // 2, 0))
+            return result
+
+    def resize2square(self, pil_img: Image.Image):
+        width, height = pil_img.size
+        pil_img = pil_img.resize((max(width, height), max(width, height)))
+        return pil_img
+
+class Processor(object):
+    def __init__(
+            self,
+            model_name_or_path,
+            max_n_frames=8,
+            max_seq_len=None,
+            add_sep=False,
+            do_image_padding=False,
+        ):
+        self.max_n_frames = max_n_frames
+        self.max_seq_len = max_seq_len,
+        self.add_sep = add_sep
+        self.do_image_padding = do_image_padding
+        if not self.do_image_padding:
+            print(f"### do_image_padding is set as False, images will be resized directly!")
+
+        self.setup(model_name_or_path)
+        
+    
+    def setup(self, model_name_or_path):
+        sub_processor = LlavaProcessor.from_pretrained(
+            model_name_or_path,
+            padding_side='left',
+            trust_remote_code=True,
+        )
+        self.processor = CustomImageProcessor(sub_processor)
+        self.tokenizer = sub_processor.tokenizer
+        # self.pad_collator = DataCollatorForSeq2Seq(self.tokenizer, padding='longest')
+        self.sep_id = self.tokenizer.sep_token_id
+        self.pad_id = self.tokenizer.pad_token_id
+        self.eos_id = self.tokenizer.eos_token_id
+
+        if self.sep_id is None:
+            self.add_sep = False
+        if not self.max_seq_len:
+            self.max_seq_len = self.tokenizer.model_max_length
+
+    def process_prompt(self, prompt, images: List[Image.Image]=None):
+        if not images:
+            prompt = prompt.replace("<image>", "").replace("<video>", "")
+        elif images is not None:
+            prompt = prompt.replace("<video>", "<image>"*len(images))
+            image_token_num = len(re.findall('<image>', prompt, re.S))
+            if image_token_num == 0:
+                prompt_parts = re.findall(r'USER:(.*)ASSISTANT:(.*)', prompt, re.S)
+                if prompt_parts and len(prompt_parts) == 2:
+                    p1, p2 = prompt_parts
+                else:
+                    p1 = prompt
+                    p2 = ''
+                prompt = f"USER: {'<image>'*len(images) + ' ' + p1.strip()} ASSISTANT: {p2.strip()}"
+            assert image_token_num == len(images)
+        
+        if not re.findall(r'USER:(.*)ASSISTANT:(.*)', prompt, re.S):
+            prompt = f'USER: {prompt} ASSISTANT: '
+        return prompt
+
+    def select_frames_sampler(self, visual_data_path):
+        visual_type = get_visual_type(visual_data_path)
+        if visual_type in ext2sampler:
+            return ext2sampler[visual_type]
+        else:
+            raise ValueError(f"Unsupported data format: {visual_data_path}")
+        
+    def load_images(self, visual_data_path, n_frames=None, start_time=0, end_time=-1):
+        sampler = self.select_frames_sampler(visual_data_path)
+        return sampler(visual_data_path, n_frames=min(n_frames, self.max_n_frames) if n_frames else self.max_n_frames, start_time=start_time, end_time=end_time)
+
+    def get_pixel_values(self, images):
+        if images is not None and len(images) > 0:
+            pixel_values = self.processor(images=images, do_padding=self.do_image_padding)
+        else:
+            pixel_values = None
+        return pixel_values
+
+    def get_text_inputs(self, text):
+        prompt_ids = self.tokenizer.encode(text, add_special_tokens=True)  # will add <s>
+        if self.add_sep:
+            prompt_ids = prompt_ids + [self.sep_id]
+        prompt_ids = torch.tensor(prompt_ids, dtype=torch.long).unsqueeze(dim=0)
+        return prompt_ids
+
+    def get_inputs(self, prompt, visual_data_file=None, images=None, n_frames=None, edit_prompt=False, return_prompt=False):
+        if images is None:
+            images = self.load_images(visual_data_file, n_frames) if visual_data_file else None
+        if edit_prompt:
+            prompt = self.process_prompt(prompt, images)
+        text_inputs = self.get_text_inputs(prompt)
+        pixel_values = self.get_pixel_values(images)
+        inputs = {
+            "input_ids": text_inputs,
+            "pixel_values": pixel_values
+        }
+        if return_prompt:
+            inputs['prompt'] = prompt
+        return inputs
+
+    def __call__(self, prompt, visual_data_file=None, images=None, n_frames=None, edit_prompt=False, return_prompt=False):
+        return self.get_inputs(prompt, visual_data_file, images, n_frames, edit_prompt, return_prompt)

dataset/utils.py

@@ -0,0 +1,128 @@
+# Copyright (2024) Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List
+import os
+from PIL import Image, ImageSequence
+import decord
+
+VALID_DATA_FORMAT_STRING = "Input data must be {'.jpg', '.jpeg', '.png', '.tif'} for image; or {'.mp4', '.avi', '.webm', '.mov', '.mkv', '.wmv', '.gif'}  for videos!"
+
+# 均匀抽帧，必采样首尾帧。
+def sample_frame_indices(start_frame, total_frames: int, n_frames: int):
+    if n_frames == 1:
+        return [0]  # sample first frame in default
+    sample_ids = [round(i * (total_frames - 1) / (n_frames - 1)) for i in range(n_frames)]
+    sample_ids = [i + start_frame for i in sample_ids]
+    return sample_ids
+
+def sample_video(
+    video_path: str, 
+    n_frames: int = None,
+    start_time: int = 0,
+    end_time: int = -1
+    ) -> List[Image.Image]:
+
+    assert os.path.exists(video_path), f"File not found: {video_path}"
+    vr = decord.VideoReader(video_path, num_threads=1, ctx=decord.cpu(0))
+    vr.seek(0)
+    total_frames = len(vr)
+    fps = vr.get_avg_fps()
+
+    start_frame = 0
+    end_frame = total_frames - 1
+    if start_time > 0:
+        start_frame = min((total_frames-1), int(fps*start_time))
+    if end_time > 0:
+        end_frame = max(start_frame, int(fps*end_time))
+        end_frame = min(end_frame, (total_frames-1))
+    frame_indices = sample_frame_indices(
+        start_frame=start_frame,
+        total_frames=end_frame - start_frame + 1,
+        n_frames=n_frames,
+    )
+
+    frames = vr.get_batch(frame_indices).asnumpy()
+    frames = [Image.fromarray(f).convert('RGB') for f in frames]
+    return frames
+
+def sample_gif(
+        gif_path: str,
+        n_frames:int = None,
+        start_time: int = 0,
+        end_time: int = -1
+    ) -> List[Image.Image]:
+
+    assert os.path.exists(gif_path), f"File not found: {gif_path}"
+    
+    gif_frames = Image.open(gif_path)
+
+    start_frame = 0
+    end_frame = gif_frames.n_frames - 1
+    frame_indices = sample_frame_indices(
+        start_frame=start_frame,
+        total_frames=end_frame - start_frame + 1,
+        n_frames=n_frames,
+    )
+        
+    frames = []
+    i = 0
+    for frame in ImageSequence.Iterator(gif_frames):
+        if i in frame_indices:
+            frames.append(frame.convert('RGB'))
+        i += 1
+    return frames
+
+def sample_image(
+    image_path: str, 
+    n_frames: int = None,
+    start_time: int = 0,
+    end_time: int = -1
+    ):
+    assert os.path.exists(image_path), f"File not found: {image_path}"
+    image = Image.open(image_path).convert('RGB')
+    return [image]
+
+def get_visual_type(input_file):
+    ext = os.path.splitext(input_file)[-1]
+    if ext in {'.gif'}:
+        return 'gif'
+    elif ext in {'.mp4', '.avi', '.webm', '.mov', '.mkv', '.wmv'}:
+        return 'video'
+    elif ext in {'.jpg', '.jpeg', '.png', '.tif'}:
+        return 'image'
+    else:
+        print(f"{VALID_DATA_FORMAT_STRING} But found {ext}!")
+        return 'unk'
+
+def get_benchmarks(benchmarks):
+    final_benchmarks = []
+    type2bm = {
+        'dream': ['dream'],
+        'caption': ['msvd-caption', 'msr-vtt-caption', 'vatex-caption'],
+        'mc_qa': ['next-qa', 'egoschema', 'mvbench', 'video-mme'],
+        'oe_qa': ['msvd-qa', 'msr-vtt-qa', 'tgif-qa', 'anet-qa'],
+    }
+    for bm in benchmarks:
+        bm = bm.lower()
+        if bm in final_benchmarks:
+            continue
+        if bm == 'all':
+            for v in type2bm.values():
+                final_benchmarks.extend(v)
+            return final_benchmarks
+        if bm in type2bm:
+            final_benchmarks.extend(type2bm[bm])
+        else:
+            final_benchmarks.append(bm)
+    return final_benchmarks

models/modeling_tarsier.py

@@ -0,0 +1,757 @@
+# Copyright (2024) Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# copy and modify from: https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py
+""" PyTorch Llava model."""
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
+import math
+import numpy as np
+
+import torch
+import torch.utils.checkpoint
+from torch import nn
+import torch.nn.functional as F
+
+from transformers import PreTrainedModel
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache
+from transformers.modeling_outputs import ModelOutput
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    logging,
+    replace_return_docstrings,
+)
+from transformers.models.auto import AutoModel, AutoModelForCausalLM, CONFIG_MAPPING
+from transformers import LlamaForCausalLM
+from transformers.configuration_utils import PretrainedConfig
+
+
+logger = logging.get_logger(__name__)
+
+LLAVA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "llava-hf/llava-v1.5-7b": "https://huggingface.co/llava-hf/llava-v1.5-7b/resolve/main/config.json",
+}
+
+class LlavaConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`LlavaForConditionalGeneration`]. It is used to instantiate an
+    Llava model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of the Llava-9B.
+
+    e.g. [llava-hf/llava-9b](https://huggingface.co/llava-hf/llava-9b)
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        vision_config (`LlavaVisionConfig`,  *optional*):
+            Custom vision config or dict
+        text_config (`Union[AutoConfig, dict]`, *optional*):
+            The config object of the text backbone. Can be any of `LlamaConfig` or `MistralConfig`.
+        ignore_index (`int`, *optional*, defaults to -100):
+            The ignore index for the loss function.
+        image_token_index (`int`, *optional*, defaults to 32000):
+            The image token index to encode the image prompt.
+        projector_hidden_act (`str`, *optional*, defaults to `"gelu"`):
+            The activation function used by the multimodal projector.
+        vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`):
+            The feature selection strategy used to select the vision feature from the CLIP backbone.
+        vision_feature_layer (`int`, *optional*, defaults to -2):
+            The index of the layer to select the vision feature.
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the Llava model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`~LlavaForConditionalGeneration`]
+
+    Example:
+
+    ```python
+    >>> from transformers import LlavaForConditionalGeneration, LlavaConfig, CLIPVisionConfig, LlamaConfig
+
+    >>> # Initializing a CLIP-vision config
+    >>> vision_config = CLIPVisionConfig()
+
+    >>> # Initializing a Llama config
+    >>> text_config = LlamaConfig()
+
+    >>> # Initializing a Llava llava-1.5-7b style configuration
+    >>> configuration = LlavaConfig(vision_config, text_config)
+
+    >>> # Initializing a model from the llava-1.5-7b style configuration
+    >>> model = LlavaForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "llava"
+    is_composition = False
+
+    def __init__(
+        self,
+        vision_config=None,
+        text_config=None,
+        ignore_index=-100,
+        image_token_index=32000,
+        projector_hidden_act="gelu",
+        vision_feature_select_strategy="default",
+        vision_feature_layer=-2,
+        vocab_size=32000,
+        image_newline_idx=32002,
+        image_new_idx=32003,
+        **kwargs,
+    ):
+        self.ignore_index = ignore_index
+        self.image_token_index = image_token_index
+        self.projector_hidden_act = projector_hidden_act
+        self.vision_feature_select_strategy = vision_feature_select_strategy
+        self.vision_feature_layer = vision_feature_layer
+        self.vocab_size = vocab_size
+        self.image_newline_idx = image_newline_idx
+        self.image_new_idx = image_new_idx
+
+        self.vision_config = vision_config
+
+        if isinstance(self.vision_config, dict):
+            vision_config["model_type"] = (
+                vision_config["model_type"] if "model_type" in vision_config else "clip_vision_model"
+            )
+            self.vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config)
+        elif vision_config is None:
+            self.vision_config = CONFIG_MAPPING["clip_vision_model"](
+                intermediate_size=4096,
+                hidden_size=1024,
+                patch_size=14,
+                image_size=336,
+                num_hidden_layers=24,
+                num_attention_heads=16,
+                vocab_size=32000,
+                projection_dim=768,
+            )
+        self.vocab_size = self.vocab_size
+
+        self.text_config = text_config
+
+        if isinstance(self.text_config, dict):
+            text_config["model_type"] = text_config["model_type"] if "model_type" in text_config else "llama"
+            self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
+            self.vocab_size = self.text_config.vocab_size
+        elif text_config is None:
+            self.text_config = CONFIG_MAPPING["llama"]()
+
+        super().__init__(**kwargs)
+
+
+logger = logging.get_logger(__name__)
+
+_CONFIG_FOR_DOC = "LlavaConfig"
+
+LLAVA_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "llava-hf/llava-1.5-7b-hf",
+    "llava-hf/llava-1.5-13b-hf",
+    "llava-hf/bakLlava-v1-hf",
+    # See all Llava models at https://huggingface.co/models?filter=llava
+]
+
+
+class Llava3DPositionalEncoding(nn.Module):
+    def __init__(self, num_pos, dim) -> None:
+        super().__init__()
+        dim1, dim2, dim3 = self.split_dim(dim)
+        frame_position_encodings = self.create_sinusoidal_positions(num_pos, dim1)
+        height_position_encodings = self.create_sinusoidal_positions(num_pos, dim2)
+        width_position_encodings = self.create_sinusoidal_positions(num_pos, dim3)
+
+        self.register_buffer('frame_position_encodings', frame_position_encodings, persistent=False)
+        self.register_buffer('height_position_encodings', height_position_encodings, persistent=False)
+        self.register_buffer('width_position_encodings', width_position_encodings, persistent=False)
+
+    def split_dim(self, dim):
+        dim1 = dim // 3
+        if dim1 % 2 != 0:
+            dim1 -= 1
+
+        dim2 = dim // 3
+        if dim2 % 2 != 0:
+            dim2 -= 1
+
+        dim3 = dim - dim1 - dim2
+        return dim1, dim2, dim3
+
+    def create_sinusoidal_positions(self, num_pos: int, dim: int) -> torch.Tensor:
+        inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim))
+        sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float()
+        return torch.cat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1)
+
+    def forward(self, frame_position_ids, height_position_ids, width_position_ids):
+        frame_position_embeds = F.embedding(frame_position_ids, self.frame_position_encodings)
+        height_position_embeds = F.embedding(height_position_ids, self.height_position_encodings)
+        width_position_embeds = F.embedding(width_position_ids, self.width_position_encodings)
+
+        return torch.cat([frame_position_embeds, height_position_embeds, width_position_embeds], dim = -1)
+
+
+@dataclass
+# Copied from transformers.models.idefics.modeling_idefics.IdeficsCausalLMOutputWithPast with Idefics->Llava
+class LlavaCausalLMOutputWithPast(ModelOutput):
+    """
+    Base class for Llava causal language model (or autoregressive) outputs.
+
+    Args:
+        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+            Language modeling loss (for next-token prediction).
+        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
+            `past_key_values` input) to speed up sequential decoding.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
+            Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
+            sequence_length, hidden_size)`.
+
+            image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
+    """
+
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    past_key_values: Optional[List[torch.FloatTensor]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+    image_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    vision_outputs: Optional[torch.FloatTensor] = None
+    llm_attn_mask: Optional[Tuple[torch.FloatTensor]] = None
+
+
+class LlavaMultiModalProjector(nn.Module):
+    def __init__(self, config: LlavaConfig):
+        super().__init__()
+
+        self.linear_1 = nn.Linear(config.vision_config.hidden_size, config.text_config.hidden_size, bias=True)
+        self.act = ACT2FN[config.projector_hidden_act]
+        self.linear_2 = nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size, bias=True)
+
+    def forward(self, image_features):
+        hidden_states = self.linear_1(image_features)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        return hidden_states
+
+
+TARSIER_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+
+    Parameters:
+        config ([`LlavaConfig`] or [`LlavaVisionConfig`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+    "The bare LLaMA Model outputting raw hidden-states without any specific head on top.",
+    TARSIER_START_DOCSTRING,
+)
+class TarsierPreTrainedModel(PreTrainedModel):
+    config_class = LlavaConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["LlavaVisionAttention"]
+    _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+
+    def _init_weights(self, module):
+        # important: this ported version of Llava isn't meant for training from scratch - only
+        # inference and fine-tuning - so the proper init weights code has been removed - the original codebase
+        # https://github.com/haotian-liu/LLaVA/tree/main/llava should serve for that purpose
+        std = (
+            self.config.initializer_range
+            if hasattr(self.config, "initializer_range")
+            else self.config.text_config.initializer_range
+        )
+
+        if hasattr(module, "class_embedding"):
+            module.class_embedding.data.normal_(mean=0.0, std=std)
+
+        if isinstance(module, (nn.Linear, nn.Conv2d)):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+    @property
+    def _supports_sdpa(self):
+        """
+        Retrieve language_model's attribute to check whether the model supports
+        SDPA or not.
+        """
+        return self.language_model._supports_sdpa
+
+
+TARSIER_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+            it.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
+            The tensors corresponding to the input images. Pixel values can be obtained using
+            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses
+            [`CLIPImageProcessor`] for processing images).
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+
+            [What are attention masks?](../glossary#attention-mask)
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
+            `past_key_values`).
+
+            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+            information on the default strategy.
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
+
+            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+    """The LLAVA model which consists of a vision backbone and a language model.""",
+    TARSIER_INPUTS_DOCSTRING,
+)
+class TarsierForConditionalGeneration(TarsierPreTrainedModel):
+    def __init__(self, config: LlavaConfig):
+        super().__init__(config)
+        self.vision_tower = AutoModel.from_config(config.vision_config, trust_remote_code=True)
+        self.multi_modal_projector = LlavaMultiModalProjector(config)
+        self.vocab_size = config.vocab_size
+        self.language_model = AutoModelForCausalLM.from_config(config.text_config, attn_implementation="flash_attention_2")
+        image_newline_idx = torch.tensor([config.image_newline_idx], dtype=torch.long)
+        image_new_idx = torch.tensor([config.image_new_idx], dtype=torch.long)
+        self.register_buffer('image_newline_idx', image_newline_idx, persistent=False)
+        self.register_buffer('image_new_idx', image_new_idx, persistent=False)
+        self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
+    def tie_weights(self):
+        return self.language_model.tie_weights()
+
+    def resize_token_embeddings(self, new_num_tokens: Optional[int] = None, pad_to_multiple_of=None) -> nn.Embedding:
+        model_embeds = self.language_model.resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
+        # update vocab size
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.config.vocab_size = model_embeds.num_embeddings
+        self.vocab_size = model_embeds.num_embeddings
+        return model_embeds
+
+    def _merge_input_ids_with_image_features(self, image_features, inputs_embeds, input_ids, attention_mask, labels):
+        num_images, num_image_patches, embed_dim = image_features.shape
+
+        batch_size, sequence_length = input_ids.shape
+        left_padding = not torch.sum(input_ids[:, -1] == torch.tensor(self.pad_token_id))
+        # 1. Create a mask to know where special image tokens are
+        special_image_token_mask = input_ids == self.config.image_token_index
+        num_special_image_tokens = torch.sum(special_image_token_mask, dim=-1)
+        # Compute the maximum embed dimension
+        max_embed_dim = (num_special_image_tokens.max() * (num_image_patches - 1)) + sequence_length
+        batch_indices, non_image_indices = torch.where(input_ids != self.config.image_token_index)
+
+        # 2. Compute the positions where text should be written
+        # Calculate new positions for text tokens in merged image-text sequence.
+        # `special_image_token_mask` identifies image tokens. Each image token will be replaced by `nb_text_tokens_per_images - 1` text tokens.
+        # `torch.cumsum` computes how each image token shifts subsequent text token positions.
+        # - 1 to adjust for zero-based indexing, as `cumsum` inherently increases indices by one.
+        new_token_positions = torch.cumsum((special_image_token_mask * (num_image_patches - 1) + 1), -1) - 1
+        nb_image_pad = max_embed_dim - 1 - new_token_positions[:, -1]
+        if left_padding:
+            new_token_positions += nb_image_pad[:, None]  # offset for left padding
+        text_to_overwrite = new_token_positions[batch_indices, non_image_indices]
+
+        # 3. Create the full embedding, already padded to the maximum position
+        final_embedding = torch.zeros(
+            batch_size, max_embed_dim, embed_dim, dtype=inputs_embeds.dtype, device=inputs_embeds.device
+        )
+        final_attention_mask = torch.zeros(
+            batch_size, max_embed_dim, dtype=attention_mask.dtype, device=inputs_embeds.device
+        )
+        if labels is not None:
+            final_labels = torch.full(
+                (batch_size, max_embed_dim), self.config.ignore_index, dtype=input_ids.dtype, device=input_ids.device
+            )
+        # In case the Vision model or the Language model has been offloaded to CPU, we need to manually
+        # set the corresponding tensors into their correct target device.
+        target_device = inputs_embeds.device
+        batch_indices, non_image_indices, text_to_overwrite = (
+            batch_indices.to(target_device),
+            non_image_indices.to(target_device),
+            text_to_overwrite.to(target_device),
+        )
+        attention_mask = attention_mask.to(target_device)
+
+        # 4. Fill the embeddings based on the mask. If we have ["hey" "<image>", "how", "are"]
+        # we need to index copy on [0, 577, 578, 579] for the text and [1:576] for the image features
+        final_embedding[batch_indices, text_to_overwrite] = inputs_embeds[batch_indices, non_image_indices]
+        final_attention_mask[batch_indices, text_to_overwrite] = attention_mask[batch_indices, non_image_indices]
+        if labels is not None:
+            final_labels[batch_indices, text_to_overwrite] = labels[batch_indices, non_image_indices]
+
+        # 5. Fill the embeddings corresponding to the images. Anything that is still zeros needs filling
+        image_to_overwrite = torch.all(final_embedding == 0, dim=-1)
+        image_to_overwrite &= image_to_overwrite.cumsum(-1) - 1 >= nb_image_pad[:, None].to(target_device)
+
+        if image_to_overwrite.sum() != image_features.shape[:-1].numel():
+            raise ValueError(
+                f"The input provided to the model are wrong. The number of image tokens is {torch.sum(special_image_token_mask)} while"
+                f" the number of image given to the model is {num_images}. This prevents correct indexing and breaks batch generation."
+            )
+
+        final_embedding[image_to_overwrite] = image_features.contiguous().reshape(-1, embed_dim).to(target_device)
+        final_attention_mask |= image_to_overwrite
+        position_ids = (final_attention_mask.cumsum(-1) - 1).masked_fill_((final_attention_mask == 0), 1)
+
+        if labels is None:
+            final_labels = None
+
+        return final_embedding, final_attention_mask, final_labels, position_ids
+    
+    def add_split_tokens(self, image_features):
+        num_images, num_image_patches, embed_dim = image_features.shape
+        num_height_patches, num_width_patches = int(math.sqrt(num_image_patches)), int(math.sqrt(num_image_patches))
+
+        # add image_newline
+        image_newline = self.get_input_embeddings()(self.image_newline_idx).squeeze()
+        image_features = image_features.view(num_images, num_height_patches, num_width_patches, embed_dim)
+        image_features = torch.cat([
+            image_features,
+            image_newline.expand((num_images, num_height_patches, 1, embed_dim)).to(device=image_features.device)
+        ], dim=2)
+        num_image_patches += num_height_patches
+        image_features = image_features.view(num_images, num_image_patches, embed_dim)
+
+        # add image_new
+        image_new = self.get_input_embeddings()(self.image_new_idx).squeeze()
+        image_features = torch.cat([
+            image_features,
+            image_new.expand((num_images, 1, embed_dim)).to(device=image_features.device)
+        ], dim = 1)
+
+        return image_features
+
+    @add_start_docstrings_to_model_forward(TARSIER_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=LlavaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        pixel_values: torch.FloatTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        vision_feature_layer: Optional[int] = None,
+        vision_feature_select_strategy: Optional[str] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ) -> Union[Tuple, LlavaCausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Returns:
+
+        Example:
+
+        ```python
+        >>> from PIL import Image
+        >>> import requests
+        >>> from transformers import AutoProcessor, LlavaForConditionalGeneration
+
+        >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
+        >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
+
+        >>> prompt = "<image>\nUSER: What's the content of the image?\nASSISTANT:"
+        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
+        >>> image = Image.open(requests.get(url, stream=True).raw)
+
+        >>> inputs = processor(text=prompt, images=image, return_tensors="pt")
+
+        >>> # Generate
+        >>> generate_ids = model.generate(**inputs, max_length=30)
+        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+        "\nUSER: What's the content of the image?\nASSISTANT: The image features a stop sign on a street corner"
+        ```"""
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        vision_feature_layer = (
+            vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer
+        )
+        vision_feature_select_strategy = (
+            vision_feature_select_strategy
+            if vision_feature_select_strategy is not None
+            else self.config.vision_feature_select_strategy
+        )
+
+        image_features = None
+        if inputs_embeds is None:
+            # 1. Extra the input embeddings
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+            
+            # 2. Merge text and images
+            if pixel_values is not None and input_ids.shape[1] != 1:
+                pixel_values = pixel_values.to(dtype=self.vision_tower.dtype)
+                image_outputs = self.vision_tower(pixel_values, output_hidden_states=True)
+                # this is not memory efficient at all (output_hidden_states=True) will save all the hidden stated.
+                selected_image_feature = image_outputs.hidden_states[vision_feature_layer]
+
+                if vision_feature_select_strategy == "default":
+                    selected_image_feature = selected_image_feature[:, 1:]
+                elif vision_feature_select_strategy == "full":
+                    selected_image_feature = selected_image_feature
+                else:
+                    raise ValueError(
+                        f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}"
+                    )
+
+                image_features = self.multi_modal_projector(selected_image_feature)
+
+                special_image_token_mask = input_ids == self.config.image_token_index
+                num_special_image_tokens = torch.sum(special_image_token_mask, dim = -1)
+
+                image_features = self.add_split_tokens(image_features)
+
+                if sum(num_special_image_tokens) > 0:
+                    # print(f'num_special_image_tokens: {num_special_image_tokens}')
+                    inputs_embeds, attention_mask, labels, position_ids = self._merge_input_ids_with_image_features(
+                        image_features, inputs_embeds, input_ids, attention_mask, labels
+                    )
+                else:
+                    inputs_embeds = image_features.sum(dim=(0,1))[None, None, :] * 0. + inputs_embeds
+
+                if labels is None:
+                    labels = torch.full_like(attention_mask, self.config.ignore_index).to(torch.long)
+            else:
+                # In case input_ids.shape[1] == 1 & pixel_values==None & past_key_values != None, we are in the case of
+                # generation with cache
+                if past_key_values is not None and pixel_values is not None and input_ids.shape[1] == 1:
+                    # Retrieve the first layer to inspect the logits and mask out the hidden states
+                    # that are set to 0
+                    first_layer_past_key_value = past_key_values[0][0][:, :, :, 0]
+
+                    # Sum all dimensions of head_dim (-2) to avoid random errors such as: https://github.com/huggingface/transformers/pull/28032#issuecomment-1863691941
+                    batch_index, non_attended_tokens = torch.where(first_layer_past_key_value.float().sum(-2) == 0)
+
+                    # Get the target length
+                    target_seqlen = first_layer_past_key_value.shape[-1] + 1
+                    extended_attention_mask = torch.ones(
+                        (attention_mask.shape[0], target_seqlen),
+                        dtype=attention_mask.dtype,
+                        device=attention_mask.device,
+                    )
+
+                    extended_attention_mask[batch_index, non_attended_tokens] = 0
+
+                    valid_indices = torch.ones_like(attention_mask)
+                    valid_indices[:, 0] = target_seqlen - extended_attention_mask.sum(dim=-1)
+                    valid_indices = torch.cumsum(valid_indices, dim=-1)
+                    extended_attention_mask = extended_attention_mask.scatter(1, valid_indices, attention_mask)
+                    attention_mask = extended_attention_mask
+                    position_ids = torch.sum(attention_mask, dim=1).unsqueeze(-1) - 1
+        outputs = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            # use_rmpad=kwargs.get("use_rmpad", False),
+            return_dict=return_dict,
+        )
+
+        logits = outputs[0]
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            if attention_mask is not None:
+                shift_attention_mask = attention_mask[..., 1:]
+                shift_logits = logits[..., :-1, :][shift_attention_mask.to(logits.device) != 0].contiguous()
+                shift_labels = labels[..., 1:][shift_attention_mask.to(labels.device) != 0].contiguous()
+            else:
+                shift_logits = logits[..., :-1, :].contiguous()
+                shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1).to(shift_logits.device)
+            )
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return LlavaCausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            llm_attn_mask=attention_mask
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, inputs_embeds=None, pixel_values=None, attention_mask=None, **kwargs
+    ):
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.seen_tokens
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+
+            # Keep only the unprocessed tokens:
+            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+            # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+            # input)
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+            # input_ids based on the past_length.
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+            elif self.config.image_token_index in input_ids:
+                input_ids = input_ids[:, input_ids.shape[1] - 1 :]
+            # If the cache has seen more tokens than it can hold, then the cache has a size limit. Let's discard the
+            # older attention values, as their corresponding values are not part of the input.
+            if cache_length < past_length and attention_mask is not None:
+                attention_mask = attention_mask[:, -(cache_length + input_ids.shape[1]) :]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "pixel_values": pixel_values,
+            }
+        )
+        return model_inputs
+
+    def _reorder_cache(self, *args, **kwargs):
+        return self.language_model._reorder_cache(*args, **kwargs)

requirements.txt

@@ -1 +1,23 @@
-huggingface_hub==0.25.2
+accelerate==1.0.1
+Pillow==9.3.0
+decord==0.6.0
+gradio==4.31.5
+ninja==1.11.1.1
+omegaconf==2.3.0
+openai==1.14.2
+pathos==0.3.2
+prettytable==3.10.0
+protobuf==3.20.3
+pycocoevalcap==1.2
+pycocotools==2.0.8
+requests==2.31.0
+safetensors==0.4.2
+scikit-learn==1.4.1.post1
+scipy==1.13.0
+tiktoken==0.6.0
+torch==2.1.0
+torchvision==0.16.0
+torchaudio==2.1.0
+https://github.com/Dao-AILab/flash-attention/releases/download/v2.5.7/flash_attn-2.5.7+cu122torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
+transformers==4.44.2
+triton==2.1.0

tools/conversation.py

@@ -0,0 +1,216 @@
+# Copyright (2024) Bytedance Ltd. and/or its affiliates
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# copy and modify from: https://github.com/OpenGVLab/Ask-Anything/blob/main/video_chat2/conversation.py
+from PIL import Image
+import torch
+from transformers import StoppingCriteria, StoppingCriteriaList
+
+from enum import auto, Enum
+import os
+from dataset.processor import Processor
+import re
+
+
+IMAGE_TOKEN = "<image>"
+VIDEO_TOKEN = "<video>"
+
+class SeparatorStyle(Enum):
+    """Different separator style."""
+    SINGLE = auto()
+    TWO = auto()
+
+def get_prompt(conv):
+    ret = ""
+    if conv.system:
+        ret = conv.system + conv.sep1
+    for i, (role, message) in enumerate(conv.messages):
+        if message:
+            # In current version, the image should be add at the first conversation round.
+            # So we need to remove the special image tokens in following user input.
+            if i > 0:
+                message = re.sub(f"({IMAGE_TOKEN}|{VIDEO_TOKEN})\n*", "", message)
+            ret += role + ": " + message
+            if i % 2:
+                ret += conv.sep2
+            else:
+                ret += conv.sep1
+        else:
+            ret += role + ":"
+    return ret
+
+
+class StoppingCriteriaSub(StoppingCriteria):
+    def __init__(self, stops=[], encounters=1):
+        super().__init__()
+        self.stops = stops
+
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor):
+        for stop in self.stops:
+            if torch.all((stop == input_ids[0][-len(stop):])).item():
+                return True
+        return False
+
+
+class Chat:
+    def __init__(self, model, processor: Processor, device='cuda', debug=False):
+        self.model = model
+        self.processor = processor
+        self.device = device
+        self.debug = debug
+        stop_words_ids = [torch.tensor([self.processor.tokenizer.eos_token_id]).to(device)]
+        self.stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops=stop_words_ids)])
+
+    def ask(self,text,conv):
+        conv.messages.append([conv.roles[0], text])
+        return conv
+
+    def prepare_model_inputs(self, conv, visual_data_file=None, images=None, n_frames=None):
+        conv.messages.append([conv.roles[1], None])
+        conv.messages[0][1] = re.sub(f"({IMAGE_TOKEN}|{VIDEO_TOKEN})\n*", "", conv.messages[0][1])
+        
+        if images is None or isinstance(images, list) and len(images) == 0:
+            if isinstance(visual_data_file, str) and os.path.exists(visual_data_file):
+                images = self.processor.load_images(visual_data_file, n_frames)
+            elif isinstance(visual_data_file, Image.Image):
+                images = [visual_data_file]
+            elif visual_data_file is None or visual_data_file == "":
+                images = None
+            else:
+                raise NotImplementedError
+        
+        if isinstance(images, list) and len(images) > 0:
+            conv.messages[0][1] = IMAGE_TOKEN*len(images) + '\n' + conv.messages[0][1]
+
+        prompt = get_prompt(conv)
+        if self.debug:
+            print(f"visual_data_file: {visual_data_file}")
+            print(f"Prompt: {prompt}", flush=True)
+
+        inputs = self.processor(prompt, images=images, edit_prompt=False, return_prompt=False)
+        inputs = {k:v.to(self.device) for k,v in inputs.items() if v is not None}
+        return inputs, conv, images
+
+    def answer(self, conv, visual_data_file=None, images=None, n_frames=None, max_new_tokens=512, num_beams=1, min_length=1, top_p=1.0,
+               repetition_penalty=1.0, length_penalty=1, temperature=0):
+        inputs, conv, images = self.prepare_model_inputs(conv, visual_data_file, images, n_frames)
+        if self.model is not None:
+            outputs = self.model.generate(
+                **inputs,
+                max_new_tokens=max_new_tokens,
+                stopping_criteria=self.stopping_criteria,
+                num_beams=num_beams,
+                do_sample=True if temperature > 0 else False,
+                min_length=min_length,
+                top_p=top_p,
+                repetition_penalty=repetition_penalty,
+                length_penalty=length_penalty,
+                temperature=temperature,
+            )
+            output_text = self.processor.tokenizer.decode(outputs[0][inputs['input_ids'][0].shape[0]:], skip_special_tokens=True)
+        else:
+            output_text = "Fake respone as launched in debug mode!"
+        conv.messages[-1][1] = output_text
+        return output_text, conv, images
+
+class EasyDict(dict):
+    """
+    Get attributes
+
+    >>> d = EasyDict({'foo':3})
+    >>> d['foo']
+    3
+    >>> d.foo
+    3
+    >>> d.bar
+    Traceback (most recent call last):
+    ...
+    AttributeError: 'EasyDict' object has no attribute 'bar'
+
+    Works recursively
+
+    >>> d = EasyDict({'foo':3, 'bar':{'x':1, 'y':2}})
+    >>> isinstance(d.bar, dict)
+    True
+    >>> d.bar.x
+    1
+    """
+
+    def __init__(self, d=None, **kwargs):
+        if d is None:
+            d = {}
+        if kwargs:
+            d.update(**kwargs)
+        for k, v in d.items():
+            setattr(self, k, v)
+        # Class attributes
+        for k in self.__class__.__dict__.keys():
+            if not (k.startswith("__") and k.endswith("__")) and not k in ("update", "pop"):
+                setattr(self, k, getattr(self, k))
+
+    def __setattr__(self, name, value):
+        if isinstance(value, (list, tuple)):
+            value = [self.__class__(x) if isinstance(x, dict) else x for x in value]
+        elif isinstance(value, dict) and not isinstance(value, self.__class__):
+            value = self.__class__(value)
+        super(EasyDict, self).__setattr__(name, value)
+        super(EasyDict, self).__setitem__(name, value)
+
+    __setitem__ = __setattr__
+
+    def update(self, e=None, **f):
+        d = e or dict()
+        d.update(f)
+        for k in d:
+            setattr(self, k, d[k])
+
+    def pop(self, k, d=None):
+        if hasattr(self, k):
+            delattr(self, k)
+        return super(EasyDict, self).pop(k, d)
+
+conv_tarsier = EasyDict({
+    "system": "",
+    "roles": ("USER", "ASSISTANT"),
+    "messages": [],
+    "sep1": " ",
+    "sep2": "</s>",
+}
+)
+
+conv_tarsier_yi = EasyDict({
+    "system": "",
+    "roles": ("USER", "ASSISTANT"),
+    "messages": [],
+    "sep1": " ",
+    "sep2": "<|endoftext|>",
+}
+)
+
+conv_tarsier_qwen2 = EasyDict({
+    "system": "",
+    "roles": ("USER", "ASSISTANT"),
+    "messages": [],
+    "sep1": " ",
+    "sep2": "<|endoftext|>",
+}
+)
+
+conv_templates = {
+    "tarsier-7b": conv_tarsier,
+    "tarsier-13b": conv_tarsier,
+    "tarsier-34b": conv_tarsier_yi,
+    "tarsier2-7b": conv_tarsier_qwen2
+}
+

tools/utils.py

@@ -0,0 +1,65 @@
+# Copyright (2024) Bytedance Ltd. and/or its affiliates
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from models.modeling_tarsier import TarsierForConditionalGeneration, LlavaConfig
+from dataset.processor import Processor
+import torch
+import base64
+
+class Color:
+
+    @staticmethod
+    def red(x):
+        return '\33[31m' +x + '\033[0m'
+    
+    @staticmethod
+    def green(x):
+        return '\33[32m' +x + '\033[0m'
+
+    @staticmethod
+    def yellow(x):
+        return '\33[33m' +x + '\033[0m'
+
+    @staticmethod
+    def blue(x):
+        return '\33[34m' +x + '\033[0m'
+
+    @staticmethod
+    def violet(x):
+        return '\33[35m' +x + '\033[0m'
+
+def file_to_base64(img_path):
+    with open(img_path, 'rb') as video_file:
+        video_b64_str = base64.b64encode(video_file.read()).decode()
+    return video_b64_str
+
+def load_model_and_processor(model_name_or_path, max_n_frames=8):
+    print(Color.red(f"Load model and processor from: {model_name_or_path}; with max_n_frames={max_n_frames}"), flush=True)
+    processor = Processor(
+        model_name_or_path,
+        max_n_frames=max_n_frames,
+    )
+    model_config = LlavaConfig.from_pretrained(
+        model_name_or_path,
+        trust_remote_code=True,
+    )
+    model = TarsierForConditionalGeneration.from_pretrained(
+        model_name_or_path,
+        config=model_config,
+        device_map='auto',
+        torch_dtype=torch.float16,
+        trust_remote_code=True
+    )
+    model.eval()
+    return model, processor
+