debug

app.py

@@ -38,7 +38,7 @@ argparser.add_argument("--model-base", type=str, default=None)
 argparser.add_argument("--num-gpus", type=int, default=1)
 argparser.add_argument("--conv-mode", type=str, default="plain")
 argparser.add_argument("--temperature", type=float, default=0.2)
-argparser.add_argument("--max-new-tokens", type=int, default=512)
+argparser.add_argument("--max-new-tokens", type=int, default=64)
 argparser.add_argument("--num_frames", type=int, default=16)
 argparser.add_argument("--load-8bit", action="store_true")
 argparser.add_argument("--load-4bit", action="store_true")
@@ -49,6 +49,7 @@ model_path = args.model_path
 conv_mode = args.conv_mode
 filt_invalid="cut"
 model_name = get_model_name_from_path(args.model_path)
+model_name = 'clip-evaclip-und-gen-pretrain'
 model_kwargs = {
     "use_cache": False,
     "trust_remote_code": True,
@@ -130,6 +131,7 @@ def generate(state, imagebox, textbox, image_process_mode, temperature, top_p, m
             images = None
             image_sizes = None
         image_args = {"images": images, "image_sizes": image_sizes}
+        print(image_args)
     else:
         images = None
         image_args = {}
@@ -140,6 +142,7 @@ def generate(state, imagebox, textbox, image_process_mode, temperature, top_p, m
     stop_str = state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2
 
     input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_IDX, return_tensors='pt').unsqueeze(0).to(model.device)
+    print(input_ids)
     streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=15)
 
     max_new_tokens = min(max_new_tokens, max_context_length - input_ids.shape[-1] - num_image_tokens)

llava/model/builder.py

@@ -48,7 +48,7 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
     if use_flash_attn:
         kwargs['attn_implementation'] = 'flash_attention_2'
 
-    if 'llava' in model_name.lower():
+    if 'und' in model_name.lower():
         # Load LLaVA model
         if 'lora' in model_name.lower() and model_base is None:
             warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument. Detailed instruction: https://github.com/haotian-liu/LLaVA#launch-a-model-worker-lora-weights-unmerged.')
@@ -157,7 +157,7 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
 
     image_processor = None
 
-    if 'llava' in model_name.lower():
+    if 'und' in model_name.lower():
         mm_use_im_start_end = getattr(model.config, "mm_use_im_start_end", False)
         mm_use_im_patch_token = getattr(model.config, "mm_use_im_patch_token", True)
         if mm_use_im_patch_token: