Upload folder using huggingface_hub

README.md

@@ -179,7 +179,7 @@ model = AutoModel.from_pretrained(
     trust_remote_code=True).eval().cuda()
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
 
-generation_config = dict(max_new_tokens=1024, do_sample=False)
+generation_config = dict(max_new_tokens=1024, do_sample=True)
 question = 'Hello, who are you?'
 response, history = model.chat(tokenizer, None, question, generation_config, history=None, return_history=True)
 print(f'User: {question}')
@@ -210,7 +210,7 @@ image_processor = CLIPImageProcessor.from_pretrained(path)
 image = Image.open('./examples/image2.jpg').resize((448, 448))
 pixel_values = image_processor(images=image, return_tensors='pt').pixel_values.to(torch.bfloat16).cuda()
 
-generation_config = dict(max_new_tokens=1024, do_sample=False)
+generation_config = dict(max_new_tokens=1024, do_sample=True)
 question = '<image>\nPlease describe the image shortly.'
 response = model.chat(tokenizer, pixel_values, question, generation_config)
 print(f'User: {question}')
@@ -236,7 +236,7 @@ image_processor = CLIPImageProcessor.from_pretrained(path)
 image = Image.open('./examples/image2.jpg').resize((448, 448))
 pixel_values = image_processor(images=image, return_tensors='pt').pixel_values.to(torch.bfloat16).cuda()
 
-generation_config = dict(max_new_tokens=1024, do_sample=False)
+generation_config = dict(max_new_tokens=1024, do_sample=True)
 question = '<image>\nPlease describe the image in detail.'
 response, history = model.chat(tokenizer, pixel_values, question, generation_config, history=None, return_history=True)
 print(f'User: {question}')
@@ -272,7 +272,7 @@ image2 = Image.open('./examples/image2.jpg').resize((448, 448))
 pixel_values2 = image_processor(images=image2, return_tensors='pt').pixel_values.to(torch.bfloat16).cuda()
 pixel_values = torch.cat((pixel_values1, pixel_values2), dim=0)
 
-generation_config = dict(max_new_tokens=1024, do_sample=False)
+generation_config = dict(max_new_tokens=1024, do_sample=True)
 question = '<image>\nDescribe the two images in detail.'
 response, history = model.chat(tokenizer, pixel_values, question, generation_config,
                                history=None, return_history=True)
@@ -311,7 +311,7 @@ pixel_values2 = image_processor(images=image2, return_tensors='pt').pixel_values
 pixel_values = torch.cat((pixel_values1, pixel_values2), dim=0)
 num_patches_list = [pixel_values1.size(0), pixel_values2.size(0)]
 
-generation_config = dict(max_new_tokens=1024, do_sample=False)
+generation_config = dict(max_new_tokens=1024, do_sample=True)
 question = 'Image-1: <image>\nImage-2: <image>\nDescribe the two images in detail.'
 response, history = model.chat(tokenizer, pixel_values, question, generation_config,
                                num_patches_list=num_patches_list, history=None, return_history=True)
@@ -348,7 +348,7 @@ pixel_values2 = image_processor(images=image2, return_tensors='pt').pixel_values
 pixel_values = torch.cat((pixel_values1, pixel_values2), dim=0)
 num_patches_list = [pixel_values1.size(0), pixel_values2.size(0)]
 
-generation_config = dict(max_new_tokens=1024, do_sample=False)
+generation_config = dict(max_new_tokens=1024, do_sample=True)
 questions = ['<image>\nDescribe the image in detail.'] * len(num_patches_list)
 responses = model.batch_chat(tokenizer, pixel_values,
                              num_patches_list=num_patches_list,
@@ -410,7 +410,7 @@ model = AutoModel.from_pretrained(
     trust_remote_code=True).eval().cuda()
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
 
-generation_config = dict(max_new_tokens=1024, do_sample=False)
+generation_config = dict(max_new_tokens=1024, do_sample=True)
 
 video_path = './examples/red-panda.mp4'
 pixel_values, num_patches_list = load_video(video_path, num_segments=8)

modeling_intern_vit.py

@@ -20,18 +20,12 @@ from transformers.utils import logging
 from .configuration_intern_vit import InternVisionConfig
 
 try:
-    try:  # v1
-        from flash_attn.flash_attn_interface import \
-            flash_attn_unpadded_qkvpacked_func
-    except:  # v2
-        from flash_attn.flash_attn_interface import \
-            flash_attn_varlen_qkvpacked_func as flash_attn_unpadded_qkvpacked_func
-
     from flash_attn.bert_padding import pad_input, unpad_input
-
+    from flash_attn.flash_attn_interface import \
+        flash_attn_varlen_qkvpacked_func
     has_flash_attn = True
 except:
-    print('FlashAttention is not installed.')
+    print('FlashAttention2 is not installed.')
     has_flash_attn = False
 
 logger = logging.get_logger(__name__)
@@ -74,7 +68,7 @@ class FlashAttention(nn.Module):
                 max_s = seqlen
                 cu_seqlens = torch.arange(0, (batch_size + 1) * seqlen, step=seqlen, dtype=torch.int32,
                                           device=qkv.device)
-                output = flash_attn_unpadded_qkvpacked_func(
+                output = flash_attn_varlen_qkvpacked_func(
                     qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
                     softmax_scale=self.softmax_scale, causal=causal
                 )
@@ -84,7 +78,7 @@ class FlashAttention(nn.Module):
                 x = rearrange(qkv, 'b s three h d -> b s (three h d)')
                 x_unpad, indices, cu_seqlens, max_s = unpad_input(x, key_padding_mask)
                 x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads)
-                output_unpad = flash_attn_unpadded_qkvpacked_func(
+                output_unpad = flash_attn_varlen_qkvpacked_func(
                     x_unpad, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
                     softmax_scale=self.softmax_scale, causal=causal
                 )
@@ -93,7 +87,7 @@ class FlashAttention(nn.Module):
                                    'b s (h d) -> b s h d', h=nheads)
         else:
             assert max_s is not None
-            output = flash_attn_unpadded_qkvpacked_func(
+            output = flash_attn_varlen_qkvpacked_func(
                 qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
                 softmax_scale=self.softmax_scale, causal=causal
             )