|
"""TraVisionLM configuration"""
|
|
|
|
from transformers import PretrainedConfig
|
|
from transformers import logging, CONFIG_MAPPING
|
|
import warnings
|
|
|
|
logger = logging.get_logger(__name__)
|
|
|
|
class TraVisionLMConfig(PretrainedConfig):
|
|
model_type = "travisionlm"
|
|
is_composition = False
|
|
|
|
def __init__(
|
|
self,
|
|
vision_config=None,
|
|
text_config=None,
|
|
ignore_index=-100,
|
|
image_token_idx=50257,
|
|
vocab_size=51282,
|
|
projection_dim=768,
|
|
hidden_size=1280,
|
|
**kwargs,
|
|
):
|
|
self.ignore_index = ignore_index
|
|
self.image_token_index = image_token_idx
|
|
self._vocab_size = vocab_size
|
|
self.projection_dim = projection_dim
|
|
self.hidden_size = hidden_size
|
|
self.vision_config = vision_config
|
|
self.is_encoder_decoder = False
|
|
if isinstance(self.vision_config, dict):
|
|
vision_config["model_type"] = (
|
|
vision_config["model_type"] if "model_type" in vision_config else "siglip_vision_model"
|
|
)
|
|
self.vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config)
|
|
elif vision_config is None:
|
|
self.vision_config = CONFIG_MAPPING["siglip_vision_model"](
|
|
attention_dropout=0.0,
|
|
hidden_act="gelu_pytorch_tanh",
|
|
hidden_size=768,
|
|
image_size=256,
|
|
intermediate_size=3072,
|
|
layer_norm_eps=1e-06,
|
|
num_attention_heads=12,
|
|
num_channels=3,
|
|
num_hidden_layers=12,
|
|
patch_size=16,
|
|
)
|
|
self.vocab_size = vocab_size
|
|
|
|
self.text_config = text_config
|
|
|
|
if isinstance(self.text_config, dict):
|
|
text_config["model_type"] = text_config["model_type"] if "model_type" in text_config else "gpt2"
|
|
self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
|
|
elif text_config is None:
|
|
self.text_config = CONFIG_MAPPING["gpt2"](
|
|
activation_function="gelu_new",
|
|
attn_pdrop=0.1,
|
|
embd_pdrop=0.1,
|
|
initializer_range=0.02,
|
|
layer_norm_epsilon=1e-05,
|
|
n_ctx=1024,
|
|
n_embd=1280,
|
|
n_head=20,
|
|
n_layer=36,
|
|
n_positions=1024,
|
|
reorder_and_upcast_attn=False,
|
|
resid_pdrop=0.1,
|
|
scale_attn_by_inverse_layer_idx=False,
|
|
scale_attn_weights=True,
|
|
vocab_size=vocab_size
|
|
)
|
|
self.num_image_tokens = (self.vision_config.image_size // self.vision_config.patch_size) ** 2
|
|
self.pad_token_id = self.text_config.pad_token_id
|
|
self.vision_config.projection_dim = projection_dim
|
|
super().__init__(**kwargs)
|
|
|
|
@property
|
|
def vocab_size(self):
|
|
warnings.warn(
|
|
"The `vocab_size` attribute is deprecated and will be removed in v4.44, Please use `text_config.vocab_size` instead.",
|
|
FutureWarning,
|
|
)
|
|
return self._vocab_size
|
|
|
|
@vocab_size.setter
|
|
def vocab_size(self, value):
|
|
self._vocab_size = value
|
|
|
|
def to_dict(self):
|
|
output = super().to_dict()
|
|
output.pop("_vocab_size", None)
|
|
return output |
