Add application file

app.py

@@ -399,12 +399,12 @@ def build_demo(embed_mode):
                 gr.Examples(
                     examples=[
                         [
-                            f"{cur_dir}/examples/extreme_ironing.jpg",
-                            "What is unusual about this image?",
+                            f"{cur_dir}/examples/CXR628_IM-2208-3001.png",
+                            "Is there any indication of an enlarged heart based on this image?",
                         ],
                         [
-                            f"{cur_dir}/examples/waterview.jpg",
-                            "What are the things I should be cautious about when I visit here?",
+                            f"{cur_dir}/examples/CXR22_IM-0810-1001.png",
+                            "nCan you identify any signs of pulmonary fibrosis?",
                         ],
                     ],
                     inputs=[imagebox, textbox],

llava/eval/eval_science_qa.py

@@ -32,6 +32,7 @@ def get_pred_idx(prediction, choices, options):
     if prediction in options[:len(choices)]:
         return options.index(prediction)
     else:
+        return -1
         return random.choice(range(len(choices)))
 
 
@@ -55,16 +56,23 @@ if __name__ == "__main__":
 
     for prob_id, prob in split_problems.items():
         if prob_id not in predictions:
-            continue
-        pred = predictions[prob_id]
-        pred_text = pred['text']
-
-        pattern = re.compile(r'The answer is ([A-Z]).')
-        res = pattern.findall(pred_text)
-        if len(res) == 1:
-            answer = res[0]  # 'A', 'B', ...
+            pred = {'text': 'FAILED', 'prompt': 'Unknown'}
+            pred_text = 'FAILED'
         else:
-            answer = "FAILED"
+            pred = predictions[prob_id]
+            pred_text = pred['text']
+
+        if pred_text in args.options:
+            answer = pred_text
+        elif len(pred_text) >= 3 and pred_text[0] in args.options and pred_text[1:3] == ". ":
+            answer = pred_text[0]
+        else:
+            pattern = re.compile(r'The answer is ([A-Z]).')
+            res = pattern.findall(pred_text)
+            if len(res) == 1:
+                answer = res[0]  # 'A', 'B', ...
+            else:
+                answer = "FAILED"
 
         pred_idx = get_pred_idx(answer, prob['choices'], args.options)
 
@@ -87,7 +95,14 @@ if __name__ == "__main__":
 
     correct = len(results['correct'])
     total = len(results['correct']) + len(results['incorrect'])
-    print(f'Total: {total}, Correct: {correct}, Accuracy: {correct / total * 100:.2f}%')
+
+    ###### IMG ######
+    multimodal_correct = len([x for x in results['correct'] if x['is_multimodal']])
+    multimodal_incorrect = len([x for x in results['incorrect'] if x['is_multimodal']])
+    multimodal_total = multimodal_correct + multimodal_incorrect
+    ###### IMG ######
+
+    print(f'Total: {total}, Correct: {correct}, Accuracy: {correct / total * 100:.2f}%, IMG-Accuracy: {multimodal_correct / multimodal_total * 100:.2f}%')
 
     sqa_results['acc'] = correct / total * 100
     sqa_results['correct'] = correct

llava/eval/model_vqa.py

@@ -66,7 +66,7 @@ def eval_model(args):
             output_ids = model.generate(
                 input_ids,
                 images=image_tensor.unsqueeze(0).half().cuda(),
-                do_sample=True,
+                do_sample=True if args.temperature > 0 else False,
                 temperature=args.temperature,
                 top_p=args.top_p,
                 num_beams=args.num_beams,

llava/eval/model_vqa_loader.py

@@ -104,7 +104,6 @@ def eval_model(args):
                 top_p=args.top_p,
                 num_beams=args.num_beams,
                 max_new_tokens=128,
-                # max_length=64,
                 use_cache=True)
 
         input_token_len = input_ids.shape[1]
@@ -124,7 +123,7 @@ def eval_model(args):
                                    "answer_id": ans_id,
                                    "model_id": model_name,
                                    "metadata": {}}) + "\n")
-        ans_file.flush()
+        # ans_file.flush()
     ans_file.close()
 
 if __name__ == "__main__":

llava/eval/model_vqa_science.py

@@ -57,6 +57,10 @@ def eval_model(args):
         else:
             images = None
 
+        if args.single_pred_prompt:
+            qs = qs + '\n' + "Answer with the option's letter from the given choices directly."
+            cur_prompt = cur_prompt + '\n' + "Answer with the option's letter from the given choices directly."
+
         conv = conv_templates[args.conv_mode].copy()
         conv.append_message(conv.roles[0], qs)
         conv.append_message(conv.roles[1], None)
@@ -72,8 +76,8 @@ def eval_model(args):
             output_ids = model.generate(
                 input_ids,
                 images=images,
-                do_sample=True,
-                temperature=0.2,
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
                 max_new_tokens=1024,
                 use_cache=True,
                 stopping_criteria=stopping_criteria,
@@ -98,8 +102,8 @@ def eval_model(args):
                 output_ids = model.generate(
                     input_ids,
                     images=images,
-                    do_sample=True,
-                    temperature=0.2,
+                    do_sample=True if args.temperature > 0 else False,
+                    temperature=args.temperature,
                     max_new_tokens=64,
                     use_cache=True,
                     stopping_criteria=[stopping_criteria])
@@ -135,7 +139,9 @@ if __name__ == "__main__":
     parser.add_argument("--conv-mode", type=str, default="llava_v0")
     parser.add_argument("--num-chunks", type=int, default=1)
     parser.add_argument("--chunk-idx", type=int, default=0)
+    parser.add_argument("--temperature", type=float, default=0.2)
     parser.add_argument("--answer-prompter", action="store_true")
+    parser.add_argument("--single-pred-prompt", action="store_true")
     args = parser.parse_args()
 
     eval_model(args)

llava/eval/summarize_gpt_review.py

@@ -9,8 +9,10 @@ import argparse
 def parse_args():
     parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
     parser.add_argument('-d', '--dir', default=None)
-    parser.add_argument('-f', '--files', nargs='*', default=None)
-    parser.add_argument('-i', '--ignore', nargs='*', default=None)
+    parser.add_argument('-v', '--version', default=None)
+    parser.add_argument('-s', '--select', nargs='*', default=None)
+    parser.add_argument('-f', '--files', nargs='*', default=[])
+    parser.add_argument('-i', '--ignore', nargs='*', default=[])
     return parser.parse_args()
 
 
@@ -20,19 +22,27 @@ if __name__ == '__main__':
     if args.ignore is not None:
         args.ignore = [int(x) for x in args.ignore]
 
-    if args.files is not None and len(args.files) > 0:
+    if len(args.files) > 0:
         review_files = args.files
     else:
-        review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and (x.startswith('gpt4_text') or x.startswith('reviews_') or x.startswith('review_'))]
+        review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and (x.startswith('gpt4_text') or x.startswith('reviews_') or x.startswith('review_') or 'review' in args.dir)]
 
     for review_file in sorted(review_files):
         config = os.path.basename(review_file).replace('gpt4_text_', '').replace('.jsonl', '')
+        if args.select is not None and any(x not in config for x in args.select):
+            continue
+        if '0613' in config:
+            version = '0613'
+        else:
+            version = '0314'
+        if args.version is not None and args.version != version:
+            continue
         scores = defaultdict(list)
         print(config)
         with open(os.path.join(args.dir, review_file) if args.dir is not None else review_file) as f:
             for review_str in f:
                 review = json.loads(review_str)
-                if args.ignore is not None and review['question_id'] in args.ignore:
+                if review['question_id'] in args.ignore:
                     continue
                 if 'category' in review:
                     scores[review['category']].append(review['tuple'])
@@ -46,5 +56,5 @@ if __name__ == '__main__':
             stats = np.asarray(v).mean(0).tolist()
             stats = [round(x, 3) for x in stats]
             # print(k, stats, round(stats[1]/stats[0]*100, 1))
-            print(k, round(stats[1]/stats[0]*100, 1))
+            print(k, round(stats[1]/stats[0]*100, 1), round(stats[0] * 10, 1), round(stats[1] * 10, 1))
         print('=================================')

llava/mm_utils.py

@@ -77,23 +77,26 @@ class KeywordsStoppingCriteria(StoppingCriteria):
     def __init__(self, keywords, tokenizer, input_ids):
         self.keywords = keywords
         self.keyword_ids = []
+        self.max_keyword_len = 0
         for keyword in keywords:
             cur_keyword_ids = tokenizer(keyword).input_ids
             if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id:
                 cur_keyword_ids = cur_keyword_ids[1:]
+            if len(cur_keyword_ids) > self.max_keyword_len:
+                self.max_keyword_len = len(cur_keyword_ids)
             self.keyword_ids.append(torch.tensor(cur_keyword_ids))
         self.tokenizer = tokenizer
         self.start_len = input_ids.shape[1]
 
     def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
         assert output_ids.shape[0] == 1, "Only support batch size 1 (yet)"  # TODO
-        offset = min(output_ids.shape[1] - self.start_len, 3)
+        offset = min(output_ids.shape[1] - self.start_len, self.max_keyword_len)
         self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids]
         for keyword_id in self.keyword_ids:
-            if output_ids[0, -keyword_id.shape[0]:] == keyword_id:
+            if (output_ids[0, -keyword_id.shape[0]:] == keyword_id).all():
                 return True
         outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0]
         for keyword in self.keywords:
             if keyword in outputs:
                 return True
-        return False
+        return False

llava/model/builder.py

@@ -23,9 +23,8 @@ from llava.model import *
 from llava.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
 
 
-def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto"):
+def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto", device="cuda"):
     kwargs = {"device_map": device_map}
-    kwargs["offload_folder"] = "offload"
 
     if load_8bit:
         kwargs['load_in_8bit'] = True
@@ -138,9 +137,7 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
         vision_tower = model.get_vision_tower()
         if not vision_tower.is_loaded:
             vision_tower.load_model()
-        
-
-        vision_tower.to(device=model.device, dtype=torch.float16)
+        vision_tower.to(device=device, dtype=torch.float16)
         image_processor = vision_tower.image_processor
 
     if hasattr(model.config, "max_sequence_length"):

llava/model/language_model/mpt/attention.py

@@ -151,7 +151,7 @@ def triton_flash_attn_fn(query, key, value, n_heads, past_key_value=None, softma
 class MultiheadAttention(nn.Module):
     """Multi-head self attention.
 
-    Using torch or triton attention implemetation enables user to also use
+    Using torch or triton attention implementation enables user to also use
     additive bias.
     """
 
@@ -204,7 +204,7 @@ class MultiheadAttention(nn.Module):
 class MultiQueryAttention(nn.Module):
     """Multi-Query self attention.
 
-    Using torch or triton attention implemetation enables user to also use
+    Using torch or triton attention implementation enables user to also use
     additive bias.
     """
 
@@ -297,4 +297,4 @@ def build_alibi_bias(n_heads, seq_len, full=False, alibi_bias_max=8, device=None
     slopes = gen_slopes(n_heads, alibi_bias_max, device=device)
     alibi_bias = alibi_bias * slopes
     return alibi_bias.to(dtype=dtype)
-ATTN_CLASS_REGISTRY = {'multihead_attention': MultiheadAttention, 'multiquery_attention': MultiQueryAttention}
+ATTN_CLASS_REGISTRY = {'multihead_attention': MultiheadAttention, 'multiquery_attention': MultiQueryAttention}

llava/model/llava_arch.py

@@ -47,12 +47,19 @@ class LlavaMetaModel:
 
         self.config.mm_vision_tower = vision_tower
 
-        vision_tower = build_vision_tower(model_args)
+        if self.get_vision_tower() is None:
+            vision_tower = build_vision_tower(model_args)
 
-        if fsdp is not None and len(fsdp) > 0:
-            self.vision_tower = [vision_tower]
+            if fsdp is not None and len(fsdp) > 0:
+                self.vision_tower = [vision_tower]
+            else:
+                self.vision_tower = vision_tower
         else:
-            self.vision_tower = vision_tower
+            if fsdp is not None and len(fsdp) > 0:
+                vision_tower = self.vision_tower[0]
+            else:
+                vision_tower = self.vision_tower
+            vision_tower.load_model()
 
         self.config.use_mm_proj = True
         self.config.mm_projector_type = getattr(model_args, 'mm_projector_type', 'linear')
@@ -60,7 +67,8 @@ class LlavaMetaModel:
         self.config.mm_vision_select_layer = mm_vision_select_layer
         self.config.mm_vision_select_feature = mm_vision_select_feature
 
-        self.mm_projector = build_vision_projector(self.config)
+        if getattr(self, 'mm_projector', None) is None:
+            self.mm_projector = build_vision_projector(self.config)
 
         if pretrain_mm_mlp_adapter is not None:
             mm_projector_weights = torch.load(pretrain_mm_mlp_adapter, map_location='cpu')

llava/serve/cli.py

@@ -5,7 +5,7 @@ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_S
 from llava.conversation import conv_templates, SeparatorStyle
 from llava.model.builder import load_pretrained_model
 from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from llava.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
 
 from PIL import Image
 
@@ -16,7 +16,7 @@ from transformers import TextStreamer
 
 
 def load_image(image_file):
-    if image_file.startswith('http') or image_file.startswith('https'):
+    if image_file.startswith('http://') or image_file.startswith('https://'):
         response = requests.get(image_file)
         image = Image.open(BytesIO(response.content)).convert('RGB')
     else:
@@ -29,7 +29,7 @@ def main(args):
     disable_torch_init()
 
     model_name = get_model_name_from_path(args.model_path)
-    tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, args.load_8bit, args.load_4bit)
+    tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, args.load_8bit, args.load_4bit, device=args.device)
 
     if 'llama-2' in model_name.lower():
         conv_mode = "llava_llama_2"
@@ -52,7 +52,12 @@ def main(args):
         roles = conv.roles
 
     image = load_image(args.image_file)
-    image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'].half().cuda()
+    # Similar operation in model_worker.py
+    image_tensor = process_images([image], image_processor, args)
+    if type(image_tensor) is list:
+        image_tensor = [image.to(model.device, dtype=torch.float16) for image in image_tensor]
+    else:
+        image_tensor = image_tensor.to(model.device, dtype=torch.float16)
 
     while True:
         try:
@@ -90,8 +95,8 @@ def main(args):
                 input_ids,
                 images=image_tensor,
                 do_sample=True,
-                temperature=0.2,
-                max_new_tokens=1024,
+                temperature=args.temperature,
+                max_new_tokens=args.max_new_tokens,
                 streamer=streamer,
                 use_cache=True,
                 stopping_criteria=[stopping_criteria])
@@ -108,12 +113,13 @@ if __name__ == "__main__":
     parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
     parser.add_argument("--model-base", type=str, default=None)
     parser.add_argument("--image-file", type=str, required=True)
-    parser.add_argument("--num-gpus", type=int, default=1)
+    parser.add_argument("--device", type=str, default="cuda")
     parser.add_argument("--conv-mode", type=str, default=None)
     parser.add_argument("--temperature", type=float, default=0.2)
     parser.add_argument("--max-new-tokens", type=int, default=512)
     parser.add_argument("--load-8bit", action="store_true")
     parser.add_argument("--load-4bit", action="store_true")
     parser.add_argument("--debug", action="store_true")
+    parser.add_argument("--image-aspect-ratio", type=str, default='pad')
     args = parser.parse_args()
     main(args)

llava/serve/model_worker.py

@@ -45,7 +45,7 @@ class ModelWorker:
     def __init__(self, controller_addr, worker_addr,
                  worker_id, no_register,
                  model_path, model_base, model_name,
-                 load_8bit, load_4bit):
+                 load_8bit, load_4bit, device):
         self.controller_addr = controller_addr
         self.worker_addr = worker_addr
         self.worker_id = worker_id
@@ -60,9 +60,10 @@ class ModelWorker:
         else:
             self.model_name = model_name
 
+        self.device = device
         logger.info(f"Loading the model {self.model_name} on worker {worker_id} ...")
         self.tokenizer, self.model, self.image_processor, self.context_len = load_pretrained_model(
-            model_path, model_base, self.model_name, load_8bit, load_4bit)
+            model_path, model_base, self.model_name, load_8bit, load_4bit, device=self.device)
         self.is_multimodal = 'llava' in self.model_name.lower()
 
         if not no_register:
@@ -159,7 +160,7 @@ class ModelWorker:
         stop_str = params.get("stop", None)
         do_sample = True if temperature > 0.001 else False
 
-        input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+        input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(self.device)
         keywords = [stop_str]
         stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
         streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=15)
@@ -258,6 +259,7 @@ if __name__ == "__main__":
     parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
     parser.add_argument("--model-base", type=str, default=None)
     parser.add_argument("--model-name", type=str)
+    parser.add_argument("--device", type=str, default="cuda")
     parser.add_argument("--multi-modal", action="store_true", help="Multimodal mode is automatically detected with model name, please make sure `llava` is included in the model path.")
     parser.add_argument("--limit-model-concurrency", type=int, default=5)
     parser.add_argument("--stream-interval", type=int, default=1)
@@ -278,5 +280,6 @@ if __name__ == "__main__":
                          args.model_base,
                          args.model_name,
                          args.load_8bit,
-                         args.load_4bit)
+                         args.load_4bit,
+                         args.device)
     uvicorn.run(app, host=args.host, port=args.port, log_level="info")

llava/train/train.py

@@ -163,12 +163,14 @@ def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match):
 def find_all_linear_names(model):
     cls = torch.nn.Linear
     lora_module_names = set()
+    multimodal_keywords = ['mm_projector', 'vision_tower', 'vision_resampler']
     for name, module in model.named_modules():
+        if any(mm_keyword in name for mm_keyword in multimodal_keywords):
+            continue
         if isinstance(module, cls):
             names = name.split('.')
             lora_module_names.add(names[0] if len(names) == 1 else names[-1])
 
-
     if 'lm_head' in lora_module_names: # needed for 16-bit
         lora_module_names.remove('lm_head')
     return list(lora_module_names)