Update processing_phi3_v.py
Browse files- processing_phi3_v.py +9 -11
processing_phi3_v.py
CHANGED
@@ -328,13 +328,13 @@ class Phi3VProcessor(ProcessorMixin):
|
|
328 |
self.img_tokens = [f"<|image_{i + 1}|>" for i in range(1000000)]
|
329 |
|
330 |
def __call__(
|
331 |
-
|
332 |
-
|
333 |
-
|
334 |
-
|
335 |
-
|
336 |
-
|
337 |
-
|
338 |
) -> BatchFeature:
|
339 |
"""
|
340 |
Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
|
@@ -415,11 +415,9 @@ class Phi3VProcessor(ProcessorMixin):
|
|
415 |
def get_special_image_token_id(self):
|
416 |
return self.tokenizer.convert_tokens_to_ids(self.special_image_token)
|
417 |
|
418 |
-
def _convert_images_texts_to_inputs(self, images, texts, padding=False, truncation=None, max_length=None,
|
419 |
-
return_tensors=None):
|
420 |
if not len(images):
|
421 |
-
model_inputs = self.tokenizer(texts, return_tensors=return_tensors, padding=padding, truncation=truncation,
|
422 |
-
max_length=max_length)
|
423 |
return BatchFeature(data={**model_inputs})
|
424 |
|
425 |
pattern = r"<\|image_\d+\|>"
|
|
|
328 |
self.img_tokens = [f"<|image_{i + 1}|>" for i in range(1000000)]
|
329 |
|
330 |
def __call__(
|
331 |
+
self,
|
332 |
+
text: Union[TextInput, List[TextInput]],
|
333 |
+
images: ImageInput = None,
|
334 |
+
padding: Union[bool, str, PaddingStrategy] = False,
|
335 |
+
truncation: Union[bool, str, TruncationStrategy] = None,
|
336 |
+
max_length=None,
|
337 |
+
return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
|
338 |
) -> BatchFeature:
|
339 |
"""
|
340 |
Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
|
|
|
415 |
def get_special_image_token_id(self):
|
416 |
return self.tokenizer.convert_tokens_to_ids(self.special_image_token)
|
417 |
|
418 |
+
def _convert_images_texts_to_inputs(self, images, texts, padding=False, truncation=None, max_length=None, return_tensors=None):
|
|
|
419 |
if not len(images):
|
420 |
+
model_inputs = self.tokenizer(texts, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length)
|
|
|
421 |
return BatchFeature(data={**model_inputs})
|
422 |
|
423 |
pattern = r"<\|image_\d+\|>"
|