xtuner/llava-phi-3-mini

Model

llava-phi-3-mini is a LLaVA model fine-tuned from microsoft/Phi-3-mini-4k-instruct and CLIP-ViT-Large-patch14-336 with ShareGPT4V-PT and InternVL-SFT by XTuner.

Note: This model is in official LLaVA format.

Resources:

• GitHub: xtuner
• HuggingFace LLaVA format model: xtuner/llava-phi-3-mini-hf
• GGUF LLaVA model: xtuner/llava-phi-3-mini-gguf
• XTuner LLaVA format model: xtuner/llava-phi-3-mini-xtuner

Details

Model	Visual Encoder	Projector	Resolution	Pretraining Strategy	Fine-tuning Strategy	Pretrain Dataset	Fine-tune Dataset	Pretrain Epoch	Fine-tune Epoch
LLaVA-v1.5-7B	CLIP-L	MLP	336	Frozen LLM, Frozen ViT	Full LLM, Frozen ViT	LLaVA-PT (558K)	LLaVA-Mix (665K)	1	1
LLaVA-Llama-3-8B	CLIP-L	MLP	336	Frozen LLM, Frozen ViT	Full LLM, LoRA ViT	LLaVA-PT (558K)	LLaVA-Mix (665K)	1	1
LLaVA-Llama-3-8B-v1.1	CLIP-L	MLP	336	Frozen LLM, Frozen ViT	Full LLM, LoRA ViT	ShareGPT4V-PT (1246K)	InternVL-SFT (1268K)	1	1
LLaVA-Phi-3-mini	CLIP-L	MLP	336	Frozen LLM, Frozen ViT	Full LLM, Full ViT	ShareGPT4V-PT (1246K)	InternVL-SFT (1268K)	1	2

Results

Model	MMBench Test (EN)	MMMU Val	SEED-IMG	AI2D Test	ScienceQA Test	HallusionBench aAcc	POPE	GQA	TextVQA	MME	MMStar
LLaVA-v1.5-7B	66.5	35.3	60.5	54.8	70.4	44.9	85.9	62.0	58.2	1511/348	30.3
LLaVA-Llama-3-8B	68.9	36.8	69.8	60.9	73.3	47.3	87.2	63.5	58.0	1506/295	38.2
LLaVA-Llama-3-8B-v1.1	72.3	37.1	70.1	70.0	72.9	47.7	86.4	62.6	59.0	1469/349	45.1
LLaVA-Phi-3-mini	69.2	41.4	70.0	69.3	73.7	49.8	87.3	61.5	57.8	1477/313	43.7

Quickstart

Chat by LLaVA official library

1. Install official LLaVA library

pip install git+https://github.com/haotian-liu/LLaVA.git

2. Chat by below script

cli.py

import argparse
from io import BytesIO

import requests
import torch
from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
from llava.conversation import Conversation, SeparatorStyle
from llava.mm_utils import process_images, tokenizer_image_token
from llava.model import LlavaLlamaForCausalLM
from PIL import Image
from transformers import (AutoTokenizer, BitsAndBytesConfig, StoppingCriteria,
                          StoppingCriteriaList, TextStreamer)


def load_image(image_file):
    if image_file.startswith('http://') or image_file.startswith('https://'):
        response = requests.get(image_file)
        image = Image.open(BytesIO(response.content)).convert('RGB')
    else:
        image = Image.open(image_file).convert('RGB')
    return image


class StopWordStoppingCriteria(StoppingCriteria):
    """StopWord stopping criteria."""

    def __init__(self, tokenizer, stop_word):
        self.tokenizer = tokenizer
        self.stop_word = stop_word
        self.length = len(self.stop_word)

    def __call__(self, input_ids, *args, **kwargs) -> bool:
        cur_text = self.tokenizer.decode(input_ids[0])
        cur_text = cur_text.replace('\r', '').replace('\n', '')
        return cur_text[-self.length:] == self.stop_word


def get_stop_criteria(tokenizer, stop_words=[]):
    stop_criteria = StoppingCriteriaList()
    for word in stop_words:
        stop_criteria.append(StopWordStoppingCriteria(tokenizer, word))
    return stop_criteria


def main(args):
    kwargs = {'device_map': args.device}
    if args.load_8bit:
        kwargs['load_in_8bit'] = True
    elif args.load_4bit:
        kwargs['load_in_4bit'] = True
        kwargs['quantization_config'] = BitsAndBytesConfig(
            load_in_4bit=True,
            bnb_4bit_compute_dtype=torch.float16,
            bnb_4bit_use_double_quant=True,
            bnb_4bit_quant_type='nf4')
    else:
        kwargs['torch_dtype'] = torch.float16

    tokenizer = AutoTokenizer.from_pretrained(args.model_path)
    model = LlavaLlamaForCausalLM.from_pretrained(
        args.model_path, low_cpu_mem_usage=True, **kwargs)
    vision_tower = model.get_vision_tower()
    if not vision_tower.is_loaded:
        vision_tower.load_model(device_map=args.device)
    image_processor = vision_tower.image_processor

    conv = Conversation(
        system=system='<|system|>\nAnswer the questions.',
        roles=('<|user|>\n', '<|assistant|>\n'),
        messages=[],
        offset=0,
        sep_style=SeparatorStyle.MPT,
        sep='<|end|>',
    )
    roles = conv.roles

    image = load_image(args.image_file)
    image_size = image.size
    image_tensor = process_images([image], image_processor, model.config)

    if type(image_tensor) is list:
        image_tensor = [
            image.to(model.device, dtype=torch.float16)
            for image in image_tensor
        ]
    else:
        image_tensor = image_tensor.to(model.device, dtype=torch.float16)

    while True:
        try:
            inp = input(f'{roles[0]}: ')
        except EOFError:
            inp = ''
        if not inp:
            print('exit...')
            break

        print(f'{roles[1]}: ', end='')

        if image is not None:
            inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
            image = None

        conv.append_message(conv.roles[0], inp)
        conv.append_message(conv.roles[1], None)
        prompt = conv.get_prompt()

        input_ids = tokenizer_image_token(
            prompt, tokenizer, IMAGE_TOKEN_INDEX,
            return_tensors='pt').unsqueeze(0).to(model.device)
        stop_criteria = get_stop_criteria(
            tokenizer=tokenizer, stop_words=[conv.sep])

        streamer = TextStreamer(
            tokenizer, skip_prompt=True, skip_special_tokens=True)

        with torch.inference_mode():
            output_ids = model.generate(
                input_ids,
                images=image_tensor,
                image_sizes=[image_size],
                do_sample=True if args.temperature > 0 else False,
                temperature=args.temperature,
                max_new_tokens=args.max_new_tokens,
                streamer=streamer,
                stopping_criteria=stop_criteria,
                use_cache=True)

        outputs = tokenizer.decode(output_ids[0]).strip()
        conv.messages[-1][-1] = outputs

        if args.debug:
            print('\n', {'prompt': prompt, 'outputs': outputs}, '\n')


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--model-path', type=str, default='xtuner/llava-llama-3-8b-v1_1-hf')
    parser.add_argument('--image-file', type=str, required=True)
    parser.add_argument('--device', type=str, default='auto')
    parser.add_argument('--temperature', type=float, default=0.2)
    parser.add_argument('--max-new-tokens', type=int, default=512)
    parser.add_argument('--load-8bit', action='store_true')
    parser.add_argument('--load-4bit', action='store_true')
    parser.add_argument('--debug', action='store_true')
    args = parser.parse_args()
    main(args)

python ./cli.py  --model-path xtuner/llava-phi-3-mini --image-file https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg  --load-4bit

Reproduce

Please refer to docs.

Citation

@misc{2023xtuner,
    title={XTuner: A Toolkit for Efficiently Fine-tuning LLM},
    author={XTuner Contributors},
    howpublished = {\url{https://github.com/InternLM/xtuner}},
    year={2023}
}