Spaces:
Build error
Build error
| # Copyright 2024 HuggingFace Inc. and the LlamaFactory team. | |
| # | |
| # This code is inspired by the HuggingFace's Transformers library. | |
| # https://github.com/huggingface/transformers/blob/v4.40.0/src/transformers/models/llava/modeling_llava.py | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| from typing import TYPE_CHECKING, Tuple | |
| import torch | |
| import transformers.models | |
| from transformers.activations import ACT2FN | |
| from transformers.utils import logging | |
| from ...extras.logging import get_logger | |
| if TYPE_CHECKING: | |
| from transformers import LlavaConfig, PretrainedConfig, PreTrainedModel | |
| from ...hparams import ModelArguments | |
| logger = get_logger(__name__) | |
| transformers_logger = logging.get_logger(__name__) | |
| class LlavaMultiModalProjectorForYiVL(torch.nn.Module): | |
| def __init__(self, config: "LlavaConfig") -> None: | |
| super().__init__() | |
| self.config = config | |
| if config is None: | |
| return | |
| self.linear_1 = torch.nn.Linear(config.vision_config.hidden_size, config.text_config.hidden_size, bias=True) | |
| self.linear_2 = torch.nn.LayerNorm(config.text_config.hidden_size, bias=True) | |
| self.linear_3 = torch.nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size, bias=True) | |
| self.linear_4 = torch.nn.LayerNorm(config.text_config.hidden_size, bias=True) | |
| self.act = ACT2FN[config.projector_hidden_act] | |
| def forward(self, image_features: "torch.Tensor") -> "torch.Tensor": | |
| hidden_states = self.linear_1(image_features) | |
| hidden_states = self.linear_2(hidden_states) | |
| hidden_states = self.act(hidden_states) | |
| hidden_states = self.linear_3(hidden_states) | |
| hidden_states = self.linear_4(hidden_states) | |
| if hidden_states.dtype == torch.float32: | |
| if torch.is_autocast_enabled(): | |
| target_dtype = torch.get_autocast_gpu_dtype() | |
| elif hasattr(self.config, "_pre_quantization_dtype"): | |
| target_dtype = self.config._pre_quantization_dtype | |
| else: | |
| target_dtype = self.linear_1.weight.dtype | |
| transformers_logger.warning_once("The hidden states seems to be silently casted in float32.") | |
| hidden_states = hidden_states.to(target_dtype) | |
| return hidden_states | |
| class LlavaMultiModalProjectorForYiVLForVLLM(LlavaMultiModalProjectorForYiVL): | |
| def __init__(self, vision_hidden_size: int, text_hidden_size: int, projector_hidden_act: str) -> None: | |
| super().__init__(config=None) | |
| self.linear_1 = torch.nn.Linear(vision_hidden_size, text_hidden_size, bias=True) | |
| self.linear_2 = torch.nn.LayerNorm(text_hidden_size, bias=True) | |
| self.linear_3 = torch.nn.Linear(text_hidden_size, text_hidden_size, bias=True) | |
| self.linear_4 = torch.nn.LayerNorm(text_hidden_size, bias=True) | |
| self.act = ACT2FN[projector_hidden_act] | |
| def autocast_projector_dtype( | |
| model: "PreTrainedModel", model_args: "ModelArguments", mm_projector_name: str = "multi_modal_projector" | |
| ) -> None: | |
| def _mm_projector_forward_post_hook( | |
| module: "torch.nn.Module", args: Tuple["torch.Tensor"], output: "torch.Tensor" | |
| ) -> "torch.Tensor": | |
| return output.to(model_args.compute_dtype) | |
| if hasattr(model, mm_projector_name) and getattr(model, "quantization_method", None): | |
| logger.info("Casting multimodal projector outputs in {}.".format(model_args.compute_dtype)) | |
| mm_projector: "torch.nn.Module" = getattr(model, mm_projector_name) | |
| mm_projector.register_forward_hook(_mm_projector_forward_post_hook) | |
| def configure_visual_model(config: "PretrainedConfig") -> None: | |
| if getattr(config, "model_type", None) == "llava": # required for ds zero3 and valuehead models | |
| setattr(config, "hidden_size", getattr(config.text_config, "hidden_size", None)) | |
| if getattr(config, "is_yi_vl_derived_model", None): | |
| logger.info("Detected Yi-VL model, applying projector patch.") | |
| transformers.models.llava.modeling_llava.LlavaMultiModalProjector = LlavaMultiModalProjectorForYiVL | |