czczup commited on
Commit
b631bf7
1 Parent(s): 3764bc2

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. README.md +5 -0
  2. modeling_internvl_chat.py +2 -1
README.md CHANGED
@@ -144,6 +144,7 @@ model = AutoModel.from_pretrained(
144
  path,
145
  torch_dtype=torch.bfloat16,
146
  low_cpu_mem_usage=True,
 
147
  trust_remote_code=True).eval().cuda()
148
  ```
149
 
@@ -158,6 +159,7 @@ model = AutoModel.from_pretrained(
158
  torch_dtype=torch.bfloat16,
159
  load_in_8bit=True,
160
  low_cpu_mem_usage=True,
 
161
  trust_remote_code=True).eval()
162
  ```
163
 
@@ -172,6 +174,7 @@ model = AutoModel.from_pretrained(
172
  torch_dtype=torch.bfloat16,
173
  load_in_4bit=True,
174
  low_cpu_mem_usage=True,
 
175
  trust_remote_code=True).eval()
176
  ```
177
 
@@ -216,6 +219,7 @@ model = AutoModel.from_pretrained(
216
  path,
217
  torch_dtype=torch.bfloat16,
218
  low_cpu_mem_usage=True,
 
219
  trust_remote_code=True,
220
  device_map=device_map).eval()
221
  ```
@@ -311,6 +315,7 @@ model = AutoModel.from_pretrained(
311
  path,
312
  torch_dtype=torch.bfloat16,
313
  low_cpu_mem_usage=True,
 
314
  trust_remote_code=True).eval().cuda()
315
  tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
316
 
 
144
  path,
145
  torch_dtype=torch.bfloat16,
146
  low_cpu_mem_usage=True,
147
+ use_flash_attn=True,
148
  trust_remote_code=True).eval().cuda()
149
  ```
150
 
 
159
  torch_dtype=torch.bfloat16,
160
  load_in_8bit=True,
161
  low_cpu_mem_usage=True,
162
+ use_flash_attn=True,
163
  trust_remote_code=True).eval()
164
  ```
165
 
 
174
  torch_dtype=torch.bfloat16,
175
  load_in_4bit=True,
176
  low_cpu_mem_usage=True,
177
+ use_flash_attn=True,
178
  trust_remote_code=True).eval()
179
  ```
180
 
 
219
  path,
220
  torch_dtype=torch.bfloat16,
221
  low_cpu_mem_usage=True,
222
+ use_flash_attn=True,
223
  trust_remote_code=True,
224
  device_map=device_map).eval()
225
  ```
 
315
  path,
316
  torch_dtype=torch.bfloat16,
317
  low_cpu_mem_usage=True,
318
+ use_flash_attn=True,
319
  trust_remote_code=True).eval().cuda()
320
  tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
321
 
modeling_internvl_chat.py CHANGED
@@ -18,7 +18,7 @@ from transformers.utils import ModelOutput, logging
18
 
19
  from .configuration_internvl_chat import InternVLChatConfig
20
  from .conversation import get_conv_template
21
- from .modeling_intern_vit import InternVisionModel
22
 
23
  logger = logging.get_logger(__name__)
24
 
@@ -49,6 +49,7 @@ class InternVLChatModel(PreTrainedModel):
49
  self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
50
  self.downsample_ratio = config.downsample_ratio
51
  self.ps_version = config.ps_version
 
52
  config.vision_config.use_flash_attn = True if use_flash_attn else False
53
  config.llm_config._attn_implementation = 'flash_attention_2' if use_flash_attn else 'eager'
54
 
 
18
 
19
  from .configuration_internvl_chat import InternVLChatConfig
20
  from .conversation import get_conv_template
21
+ from .modeling_intern_vit import InternVisionModel, has_flash_attn
22
 
23
  logger = logging.get_logger(__name__)
24
 
 
49
  self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
50
  self.downsample_ratio = config.downsample_ratio
51
  self.ps_version = config.ps_version
52
+ use_flash_attn = use_flash_attn if has_flash_attn else False
53
  config.vision_config.use_flash_attn = True if use_flash_attn else False
54
  config.llm_config._attn_implementation = 'flash_attention_2' if use_flash_attn else 'eager'
55