|
from transformers import PretrainedConfig |
|
from transformers import CONFIG_MAPPING |
|
from transformers import AutoConfig |
|
|
|
IGNORE_INDEX = -100 |
|
IMAGE_TOKEN_INDEX = -200 |
|
DEFAULT_IMAGE_TOKEN = "<image>" |
|
|
|
|
|
class TinyLlavaConfig(PretrainedConfig): |
|
|
|
model_type = "tinyllava" |
|
def __init__( |
|
self, |
|
llm_model_name_or_path = '', |
|
tokenizer_name_or_path = None, |
|
vision_model_name_or_path = '', |
|
vision_model_name_or_path2 = '', |
|
connector_type = None, |
|
text_config=None, |
|
hidden_size=2048, |
|
vocab_size=32000, |
|
ignore_index=-100, |
|
image_token_index=32000, |
|
pad_token = None, |
|
pad_token_id = None, |
|
tokenizer_padding_side = 'right', |
|
tokenizer_model_max_length = 2048, |
|
vision_config = None, |
|
vision_hidden_size = None, |
|
vision_feature_layer = -2, |
|
vision_feature_select_strategy = 'patch', |
|
image_aspect_ratio = 'square', |
|
resampler_hidden_size = None, |
|
num_queries = None, |
|
num_resampler_layers = None, |
|
use_cache = False, |
|
cache_dir = None, |
|
tokenizer_use_fast = False, |
|
tune_type_llm = 'frozen', |
|
tune_type_connector = 'frozen', |
|
tune_type_vision_tower = 'frozen', |
|
tune_vision_tower_from_layer = -1, |
|
|
|
**kwargs |
|
|
|
): |
|
self.llm_model_name_or_path = llm_model_name_or_path |
|
self.tokenizer_name_or_path = tokenizer_name_or_path or self.llm_model_name_or_path |
|
self.vision_model_name_or_path = vision_model_name_or_path |
|
self.vision_model_name_or_path2 = vision_model_name_or_path2 |
|
self.connector_type = connector_type |
|
self.tune_type_llm = tune_type_llm |
|
self.tune_type_connector = tune_type_connector |
|
self.tune_type_vision_tower = tune_type_vision_tower |
|
self.tune_vision_tower_from_layer = tune_vision_tower_from_layer |
|
|
|
self.ignore_index = IGNORE_INDEX |
|
self.image_token_index = IMAGE_TOKEN_INDEX |
|
self.pad_token = pad_token |
|
self.pad_token_id = pad_token_id |
|
self.tokenizer_padding_side = tokenizer_padding_side |
|
self.tokenizer_model_max_length = tokenizer_model_max_length |
|
self.vision_feature_layer = vision_feature_layer |
|
self.vision_feature_select_strategy = vision_feature_select_strategy |
|
self.image_aspect_ratio = image_aspect_ratio |
|
self.resampler_hidden_size = resampler_hidden_size |
|
self.num_queries = num_queries |
|
self.num_resampler_layers = num_resampler_layers |
|
self.use_cache = use_cache |
|
self.cache_dir = cache_dir |
|
self.tokenizer_use_fast = tokenizer_use_fast |
|
self._load_text_config(text_config) |
|
self._load_vision_config(vision_config) |
|
|
|
super().__init__(**kwargs) |
|
|
|
|
|
def _load_text_config(self, text_config=None): |
|
if self.llm_model_name_or_path is None or self.llm_model_name_or_path == '': |
|
self.text_config = CONFIG_MAPPING['llama']() |
|
|
|
else: |
|
self.text_config = AutoConfig.from_pretrained(self.llm_model_name_or_path, trust_remote_code=True) |
|
if text_config is not None: |
|
self.text_config = self.text_config.from_dict(text_config) |
|
|
|
self.hidden_size = getattr(self.text_config, 'hidden_size', getattr(self.text_config, 'model_dim', None)) |
|
self.vocab_size = getattr(self.text_config, 'vocab_size', None) |
|
|
|
|
|
|
|
def _load_vision_config(self, vision_config=None): |
|
if self.vision_model_name_or_path is None or self.vision_model_name_or_path == '': |
|
self.vision_config = CONFIG_MAPPING['clip_vision_model']( |
|
intermediate_size=4096, |
|
hidden_size=1024, |
|
patch_size=14, |
|
image_size=336, |
|
num_hidden_layers=24, |
|
num_attention_heads=16, |
|
vocab_size=32000, |
|
projection_dim=768, |
|
) |
|
|
|
else: |
|
self.vision_config = AutoConfig.from_pretrained(self.vision_model_name_or_path.split(':')[-1]) |
|
self.vision_config = getattr(self.vision_config, 'vision_config', self.vision_config) |
|
if vision_config is not None: |
|
self.vision_config = self.vision_config.from_dict(vision_config) |
|
|
|
self.vision_config.model_name_or_path = self.vision_model_name_or_path.split(':')[-1] |
|
self.vision_config.model_name_or_path2 = self.vision_model_name_or_path2.split(':')[-1] |
|
self.vision_hidden_size = getattr(self.vision_config, 'hidden_size', None) |
|
|
|
|
|
|