Update model files

processing_llava.py

@@ -30,6 +30,7 @@ from transformers.tokenization_utils_base import (
 from transformers.utils import TensorType
 import torch
 from open_clip.transform import PreprocessCfg, image_transform_v2
+from modeling_llava import LlavaForConditionalGeneration
 
 
 class OpenCLIPImageProcessor:
@@ -67,6 +68,7 @@ class LlavaProcessor:
             TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]
         ] = None,
         images: ImageInput = None,
+        model: LlavaForConditionalGeneration = None,
         padding: Union[bool, str, PaddingStrategy] = False,
         truncation: Union[bool, str, TruncationStrategy] = None,
         max_length=None,
@@ -76,8 +78,11 @@ class LlavaProcessor:
             pixel_values = self.image_processor(images, return_tensors=return_tensors)[
                 "pixel_values"
             ]
+            pixel_values = pixel_values.to(model.device)
+            image_outputs = model.vision_model(pixel_values)
+            image_features = model.multi_modal_projector(image_outputs)
         else:
-            pixel_values = None
+            image_features = None
         text_inputs = self.tokenizer(
             text,
             return_tensors=return_tensors,
@@ -86,7 +91,7 @@ class LlavaProcessor:
             max_length=max_length,
         )
 
-        return BatchFeature(data={**text_inputs, "pixel_values": pixel_values})
+        return BatchFeature(data={**text_inputs, "image_features": image_features})
 
     def batch_decode(self, *args, **kwargs):
         return self.tokenizer.batch_decode(*args, **kwargs)