update inference code to support transformers==4.41.1

README.md

@@ -52,7 +52,7 @@ More technical details will come with a technical report soon.
 
 # How to use
 
-> We require the use of the development version (`"4.41.0.dev0"`) of the `transformers` library. To get it, as of 05/07/2024, one can use `pip uninstall -y transformers && pip install git+https://github.com/huggingface/transformers.`
+~~> We require the use of the development version (`"4.41.0.dev0"`) of the `transformers` library. To get it, as of 05/07/2024, one can use `pip uninstall -y transformers && pip install git+https://github.com/huggingface/transformers.`~~
 
 ```python
 from transformers import AutoModelForVision2Seq, AutoTokenizer, AutoImageProcessor, StoppingCriteria
@@ -149,4 +149,10 @@ pip install torch==2.2.1 torchvision==0.17.1 torchaudio==2.2.1 --index-url https
 pip install open_clip_torch==2.24.0
 pip install einops
 pip install einops-exts
-```
+pip install transformers==4.41.1
+```
+
+# Changelog
+
+* 05/24/2024
+    * update codebase to be compatiable with `transformers==4.41.1`.

config.json

@@ -1,12 +1,12 @@
 {
   "architectures": [
-    "Blip3ModelForConditionalGeneration"
+    "XGenMMModelForConditionalGeneration"
   ],
   "auto_map": {
-    "AutoConfig": "configuration_blip_3.Blip3Config",
-    "AutoModelForVision2Seq": "modeling_blip_3.Blip3ModelForConditionalGeneration"
+    "AutoConfig": "configuration_xgenmm.XGenMMConfig",
+    "AutoModelForVision2Seq": "modeling_xgenmm.XGenMMModelForConditionalGeneration"
   },
-  "model_type": "blip_3",
+  "model_type": "xgenmm",
   "text_config": {
     "initial_tokenizer_len": 32012,
     "model_type": "phi3",
@@ -14,13 +14,13 @@
     "torch_dtype": "bfloat16"
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.41.0.dev0",
+  "transformers_version": "4.41.1",
   "vision_encoder_config": {
     "anyres_patch_sampling": true,
     "image_aspect_ratio": "anyres",
-    "model_type": "blip_3_vision_encoder"
+    "model_type": "xgenmm_vision_encoder"
   },
   "vision_tokenizer_config": {
-    "model_type": "blip_3_vision_tokenizer"
+    "model_type": "xgenmm_vision_tokenizer"
   }
 }

configuration_blip_3.py → configuration_xgenmm.py

@@ -4,8 +4,8 @@ from transformers import CONFIG_MAPPING
 
 logger = logging.get_logger(__name__)
 
-class Blip3VisionEncoderConfig(PretrainedConfig):
-    model_type = "blip_3_vision_encoder"
+class XGenMMVisionEncoderConfig(PretrainedConfig):
+    model_type = "xgenmm_vision_encoder"
     
     def __init__(self, 
                  model_name: str = 'ViT-H-14-378-quickgelu',
@@ -16,8 +16,8 @@ class Blip3VisionEncoderConfig(PretrainedConfig):
         super().__init__(**kwargs)
     
 
-class Blip3VisionTokenizerConfig(PretrainedConfig):
-    model_type = "blip_3_vision_tokenizer"
+class XGenMMVisionTokenizerConfig(PretrainedConfig):
+    model_type = "xgenmm_vision_tokenizer"
     
     def __init__(self, 
                  vis_feature_dim: int = 1280,
@@ -34,8 +34,8 @@ class Blip3VisionTokenizerConfig(PretrainedConfig):
         super().__init__(**kwargs)
         
         
-class Blip3Config(PretrainedConfig):
-    model_type = "blip_3"
+class XGenMMConfig(PretrainedConfig):
+    model_type = "xgenmm"
     
     def __init__(self, 
                  vision_encoder_config: dict = None,
@@ -45,11 +45,11 @@ class Blip3Config(PretrainedConfig):
         
         if vision_encoder_config is None:
             vision_encoder_config = {'image_aspect_ratio': 'anyres', 'anyres_patch_sampling': True}
-            logger.info("vision_encoder_config is None. initializing the Blip3VisionEncoderConfig with default values.")
+            logger.info("vision_encoder_config is None. initializing the XGenMMVisionEncoderConfig with default values.")
 
         if vision_tokenizer_config is None:
             vision_tokenizer_config = {}
-            logger.info("vision_tokenizer_config is None. Initializing the Blip3VisionTokenizerConfig with default values.")
+            logger.info("vision_tokenizer_config is None. Initializing the XGenMMVisionTokenizerConfig with default values.")
 
         if text_config is None:
             text_config = {
@@ -131,9 +131,9 @@ class Blip3Config(PretrainedConfig):
                 }
             logger.info("text_config is None. Initializing the text config with default values (`Phi3Config`).")
         
-        self.vision_encoder_config = Blip3VisionEncoderConfig(**vision_encoder_config)
+        self.vision_encoder_config = XGenMMVisionEncoderConfig(**vision_encoder_config)
         
-        self.vision_tokenizer_config = Blip3VisionTokenizerConfig(**vision_tokenizer_config)
+        self.vision_tokenizer_config = XGenMMVisionTokenizerConfig(**vision_tokenizer_config)
         
         text_model_type = text_config["model_type"] if "model_type" in text_config else "phi3"
         self.text_config = CONFIG_MAPPING[text_model_type](**text_config)
@@ -147,8 +147,8 @@ class Blip3Config(PretrainedConfig):
     @classmethod
     def from_vision_encoder_vision_tokenizer_text_configs(
         cls,
-        vision_encoder_config: Blip3VisionEncoderConfig,
-        vision_tokenizer_config: Blip3VisionTokenizerConfig,
+        vision_encoder_config: XGenMMVisionEncoderConfig,
+        vision_tokenizer_config: XGenMMVisionTokenizerConfig,
         text_config: PretrainedConfig,
         **kwargs):
         

demo.ipynb

@@ -2,21 +2,44 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0585fe10e4854d99857d74e836379a47",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
+     ]
+    }
+   ],
    "source": [
     "from transformers import AutoModelForVision2Seq, AutoTokenizer, AutoImageProcessor, StoppingCriteria\n",
     "import torch\n",
-    "model = AutoModelForVision2Seq.from_pretrained(\"Salesforce/blip3-phi3-mini-instruct-r-v1\", trust_remote_code=True)\n",
-    "tokenizer = AutoTokenizer.from_pretrained(\"Salesforce/blip3-phi3-mini-instruct-r-v1\", trust_remote_code=True, use_fast=False, legacy=False)\n",
-    "image_processor = AutoImageProcessor.from_pretrained(\"Salesforce/blip3-phi3-mini-instruct-r-v1\", trust_remote_code=True)\n",
+    "model_name_or_path = \"Salesforce/xgen-mm-phi3-mini-instruct-r-v1\"\n",
+    "model = AutoModelForVision2Seq.from_pretrained(model_name_or_path, trust_remote_code=True)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=False, legacy=False)\n",
+    "image_processor = AutoImageProcessor.from_pretrained(model_name_or_path, trust_remote_code=True)\n",
     "tokenizer = model.update_special_tokens(tokenizer)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -46,17 +69,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
     "model = model.to('cuda')\n",
-    "model.eval()"
+    "model.eval()\n",
+    "tokenizer.padding_side = \"left\""
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -73,6 +97,13 @@
      },
      "output_type": "display_data"
     },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "You are not running the flash-attention implementation, expect numerical differences.\n"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -223,7 +254,6 @@
     }
    ],
    "source": [
-    "tokenizer.padding_side = \"left\"\n",
     "for sample in data:\n",
     "    img = PIL.Image.open(sample['image_path'])\n",
     "    display.display(Image(filename=sample['image_path'], width=300))\n",
@@ -262,7 +292,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.19"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 1,
   "eos_token_id": 32000,
   "pad_token_id": 32000,
-  "transformers_version": "4.41.0.dev0"
+  "transformers_version": "4.41.1"
 }

modeling_blip_3.py → modeling_xgenmm.py

@@ -4,13 +4,13 @@ import open_clip
 from typing import List, Optional, Tuple, Union
 from .utils import check_embedding_fns
 from .vlm import InstructPerceiverResampler, KosmosInstruct
-from .configuration_blip_3 import Blip3VisionEncoderConfig, Blip3VisionTokenizerConfig, Blip3Config
+from .configuration_xgenmm import XGenMMVisionEncoderConfig, XGenMMVisionTokenizerConfig, XGenMMConfig
 
-class Blip3VisionEncoder(PreTrainedModel):
+class XGenMMVisionEncoder(PreTrainedModel):
     main_input_name = "pixel_values"
-    config_class = Blip3VisionEncoderConfig
+    config_class = XGenMMVisionEncoderConfig
     
-    def __init__(self, config: Blip3VisionEncoderConfig):
+    def __init__(self, config: XGenMMVisionEncoderConfig):
         super().__init__(config)
         if config.model_name != 'ViT-H-14-378-quickgelu':
             raise ValueError(f"Unsupported model {config.model_name}. New vision models will be added soon.")
@@ -25,9 +25,9 @@ class Blip3VisionEncoder(PreTrainedModel):
         
 
 # vision tokenizer        
-class Blip3VisionTokenizer(PreTrainedModel):
-    config_class = Blip3VisionTokenizerConfig
-    def __init__(self, config: Blip3VisionTokenizerConfig):
+class XGenMMVisionTokenizer(PreTrainedModel):
+    config_class = XGenMMVisionTokenizerConfig
+    def __init__(self, config: XGenMMVisionTokenizerConfig):
         super().__init__(config)
         self.model = InstructPerceiverResampler(
             dim_llm=config.lang_embedding_dim,
@@ -42,15 +42,15 @@ class Blip3VisionTokenizer(PreTrainedModel):
                 vision_attn_masks: torch.Tensor):
         return self.model(vision_features, vision_attn_masks)
     
-# Blip3 model
-class Blip3ModelForConditionalGeneration(PreTrainedModel):
-    config_class = Blip3Config
+# XGenMM model
+class XGenMMModelForConditionalGeneration(PreTrainedModel):
+    config_class = XGenMMConfig
     
-    def __init__(self, config: Blip3Config):
+    def __init__(self, config: XGenMMConfig):
         super().__init__(config)
         
         # vision encoder initialization
-        vision_encoder = Blip3VisionEncoder(config.vision_encoder_config).model
+        vision_encoder = XGenMMVisionEncoder(config.vision_encoder_config).model
         vision_encoder.visual.output_tokens = True
         vision_encoder = vision_encoder.visual
         
@@ -67,7 +67,7 @@ class Blip3ModelForConditionalGeneration(PreTrainedModel):
             config.vision_tokenizer_config.lang_embedding_dim = overwrite
             print(f"Warning: The language embedding dimension in the vision tokenizer config is different from the language model's embedding dimension. Overwriting the language embedding dimension in the vision tokenizer config to {overwrite}.")
             
-        vision_tokenizer = Blip3VisionTokenizer(config.vision_tokenizer_config).model
+        vision_tokenizer = XGenMMVisionTokenizer(config.vision_tokenizer_config).model
 
         self.vlm = KosmosInstruct(
             vision_encoder=vision_encoder,

setup.sh

@@ -0,0 +1,7 @@
+pip install torch==2.2.1 torchvision==0.17.1 torchaudio==2.2.1 --index-url https://download.pytorch.org/whl/cu121
+pip install open_clip_torch==2.24.0
+pip install einops
+pip install einops-exts
+pip install transformers==4.41.1
+# optional
+pip install ipywidgets

utils.py

@@ -2,7 +2,7 @@ import torch
 import ast
 import math
 from PIL import Image
-
+from packaging.version import Version
 
 def has_fn(model, fn_name):
     """Check if model has a function fn_name"""

vlm.py

@@ -10,6 +10,7 @@ from transformers.modeling_outputs import CausalLMOutputWithPast
 from dataclasses import dataclass
 from transformers import CLIPVisionModel
 import transformers
+from packaging.version import Version
 
 from .utils import num_params, getattr_recursive, stack_with_padding, get_anyres_image_grid_shape, unpad_image
 
@@ -1512,7 +1513,7 @@ class KosmosInstruct(VLMWithLanguageStream):
             padding_side="left",
             num_beams=num_beams,
         )
-        if transformers.__version__ == '4.41.0.dev0':
+        if Version(transformers.__version__) >= Version('4.41.1'):
             output = self.lang_model.generate(
                 **new_inputs,
                 num_beams=num_beams,
@@ -1520,12 +1521,7 @@ class KosmosInstruct(VLMWithLanguageStream):
                 **kwargs,
             )
         else:
-            output = self.lang_model.generate(
-                **new_inputs,
-                past_key_values=past_key_values,
-                num_beams=num_beams,
-                use_cache=True,
-                **kwargs,
-            )
+            raise ValueError("Please upgrade transformers to version 4.41.1 or higher.")
+            
         self._post_forward_hook()
         return output