add app file

.gitignore

@@ -0,0 +1,37 @@
+# Python
+__pycache__
+*.pyc
+*.egg-info
+dist
+
+# Log
+*.log
+*.log.*
+*.jsonl
+
+# Data
+!**/alpaca-data-conversation.json
+
+# Editor
+.idea
+*.swp
+
+# Other
+.DS_Store
+wandb
+output
+work_dirs
+data
+model_zoo
+
+checkpoints
+ckpts*
+
+.ipynb_checkpoints
+*.ipynb
+
+# DevContainer
+!.devcontainer/*
+
+# Demo
+serve_images/

app.py

@@ -0,0 +1,339 @@
+import shutil
+import subprocess
+
+import timm
+import spaces
+import io
+import base64
+
+import torch
+import gradio as gr
+import os
+from PIL import Image
+import tempfile
+from huggingface_hub import snapshot_download
+from transformers import TextIteratorStreamer
+from threading import Thread
+
+from diffusers import StableDiffusionXLPipeline
+
+from minigemini.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
+from minigemini.mm_utils import process_images, load_image_from_base64, tokenizer_image_token
+from minigemini.conversation import default_conversation, conv_templates, SeparatorStyle, Conversation
+from minigemini.serve.gradio_web_server import function_markdown, tos_markdown, learn_more_markdown, title_markdown, block_css
+from minigemini.model.builder import load_pretrained_model
+
+# os.system('python -m pip install paddlepaddle-gpu==2.4.2.post117 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html')
+# os.system('pip install paddleocr>=2.0.1')
+# from paddleocr import PaddleOCR
+
+def download_model(repo_id):
+    local_dir = os.path.join('./checkpoints', repo_id.split('/')[-1])
+    os.makedirs(local_dir)
+    snapshot_download(repo_id=repo_id, local_dir=local_dir, local_dir_use_symlinks=False)
+
+
+if not os.path.exists('./checkpoints/'):
+    os.makedirs('./checkpoints/')
+download_model('YanweiLi/Mini-Gemini-13B-HD')
+download_model('laion/CLIP-convnext_large_d_320.laion2B-s29B-b131K-ft-soup')
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+load_8bit = False
+load_4bit = False
+dtype = torch.float16
+conv_mode = "vicuna_v1"
+model_path = './checkpoints/Mini-Gemini-13B-HD'
+model_name = 'Mini-Gemini-13B-HD'
+model_base = None
+
+tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, model_base, model_name,
+                                                                 load_8bit, load_4bit,
+                                                                 device=device)
+
+diffusion_pipe = StableDiffusionXLPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0",
+    torch_dtype=torch.float16,
+    use_safetensors=True, variant="fp16"
+).to(device=device)
+
+
+if hasattr(model.config, 'image_size_aux'):
+    if not hasattr(image_processor, 'image_size_raw'):
+        image_processor.image_size_raw = image_processor.crop_size.copy()
+    image_processor.crop_size['height'] = model.config.image_size_aux
+    image_processor.crop_size['width'] = model.config.image_size_aux
+    image_processor.size['shortest_edge'] = model.config.image_size_aux
+
+no_change_btn = gr.Button()
+enable_btn = gr.Button(interactive=True)
+disable_btn = gr.Button(interactive=False)
+
+
+def upvote_last_response(state):
+    return ("",) + (disable_btn,) * 3
+
+def downvote_last_response(state):
+    return ("",) + (disable_btn,) * 3
+
+def flag_last_response(state):
+    return ("",) + (disable_btn,) * 3
+
+def clear_history():
+    state = conv_templates[conv_mode].copy()
+    return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
+
+
+def process_image(prompt, images):
+    if images is not None and len(images) > 0:
+        image_convert = images
+        
+        # Similar operation in model_worker.py
+        image_tensor = process_images(image_convert, image_processor, model.config)
+    
+        image_grid = getattr(model.config, 'image_grid', 1)
+        if hasattr(model.config, 'image_size_aux'):
+            raw_shape = [image_processor.image_size_raw['height'] * image_grid,
+                         image_processor.image_size_raw['width'] * image_grid]
+            image_tensor_aux = image_tensor 
+            image_tensor = torch.nn.functional.interpolate(image_tensor,
+                                                        size=raw_shape,
+                                                        mode='bilinear',
+                                                        align_corners=False)
+        else:
+            image_tensor_aux = []
+
+        if image_grid >= 2:            
+            raw_image = image_tensor.reshape(3, 
+                                            image_grid,
+                                            image_processor.image_size_raw['height'],
+                                            image_grid,
+                                            image_processor.image_size_raw['width'])
+            raw_image = raw_image.permute(1, 3, 0, 2, 4)
+            raw_image = raw_image.reshape(-1, 3,
+                                        image_processor.image_size_raw['height'],
+                                        image_processor.image_size_raw['width'])
+                    
+            if getattr(model.config, 'image_global', False):
+                global_image = image_tensor
+                if len(global_image.shape) == 3:
+                    global_image = global_image[None]
+                global_image = torch.nn.functional.interpolate(global_image, 
+                                                            size=[image_processor.image_size_raw['height'],
+                                                                    image_processor.image_size_raw['width']], 
+                                                            mode='bilinear', 
+                                                            align_corners=False)
+                # [image_crops, image_global]
+                raw_image = torch.cat([raw_image, global_image], dim=0)
+            image_tensor = raw_image.contiguous()
+            image_tensor = image_tensor.unsqueeze(0)
+    
+        if type(image_tensor) is list:
+            image_tensor = [image.to(model.device, dtype=torch.float16) for image in image_tensor]
+            image_tensor_aux = [image.to(model.device, dtype=torch.float16) for image in image_tensor_aux]
+        else:
+            image_tensor = image_tensor.to(model.device, dtype=torch.float16)
+            image_tensor_aux = image_tensor_aux.to(model.device, dtype=torch.float16)
+    else:
+        images = None
+        image_tensor = None
+        image_tensor_aux = []
+    
+    image_tensor_aux = image_tensor_aux if len(image_tensor_aux) > 0 else None
+
+    replace_token = DEFAULT_IMAGE_TOKEN
+    if getattr(model.config, 'mm_use_im_start_end', False):
+        replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN
+    prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token)
+
+    image_args = {"images": image_tensor, "images_aux": image_tensor_aux}
+
+    return prompt, image_args
+
+
+@spaces.GPU
+def generate(state, imagebox, textbox, image_process_mode, gen_image, temperature, top_p, max_output_tokens):
+    prompt = state.get_prompt()
+    images = state.get_images(return_pil=True)
+    prompt, image_args = process_image(prompt, images)
+
+    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to("cuda:0")
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=30)
+
+    max_new_tokens = 512
+    do_sample = True if temperature > 0.001 else False
+    stop_str = state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2
+
+    thread = Thread(target=model.generate, kwargs=dict(
+        inputs=input_ids,
+        do_sample=do_sample,
+        temperature=temperature,
+        top_p=top_p,
+        max_new_tokens=max_new_tokens,
+        streamer=streamer,
+        use_cache=True,
+        **image_args
+    ))
+    thread.start()
+
+    generated_text = ''
+    for new_text in streamer:
+        generated_text += new_text
+        if generated_text.endswith(stop_str):
+            generated_text = generated_text[:-len(stop_str)]
+        state.messages[-1][-1] = generated_text
+        yield (state, state.to_gradio_chatbot(), "", None) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+    
+    if gen_image == 'Yes' and '<h>' in generated_text and '</h>' in generated_text:
+        common_neg_prompt = "out of frame, lowres, text, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature"
+        prompt = generated_text.split("<h>")[1].split("</h>")[0]
+        generated_text = generated_text.split("<h>")[0] + '\n' + 'Prompt: ' + prompt + '\n'
+
+        torch.cuda.empty_cache()
+        output_img = diffusion_pipe(prompt, negative_prompt=common_neg_prompt).images[0]
+        buffered = io.BytesIO()
+        output_img.save(buffered, format='JPEG')
+        img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+
+        output = (generated_text, img_b64_str)
+        state.messages[-1][-1] = output
+    
+    yield (state, state.to_gradio_chatbot(), "", None) + (enable_btn,) * 5
+    
+    torch.cuda.empty_cache()
+
+
+@spaces.GPU
+def add_text(state, imagebox, textbox, image_process_mode, gen_image):
+    if state is None:
+        state = conv_templates[conv_mode].copy()
+
+    if imagebox is not None:
+        textbox = DEFAULT_IMAGE_TOKEN + '\n' + textbox
+        image = Image.open(imagebox).convert('RGB')
+
+    if gen_image == 'Yes':
+        textbox = textbox + ' <GEN>'
+
+    if imagebox is not None:
+        textbox = (textbox, image, image_process_mode)
+
+    state.append_message(state.roles[0], textbox)
+    state.append_message(state.roles[1], None)
+
+    yield (state, state.to_gradio_chatbot(), "", None) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+
+
+def delete_text(state, image_process_mode):
+    state.messages[-1][-1] = None
+    prev_human_msg = state.messages[-2]
+    if type(prev_human_msg[1]) in (tuple, list):
+        prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)
+    yield (state, state.to_gradio_chatbot(), "", None) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
+
+
+textbox = gr.Textbox(show_label=False, placeholder="Enter text and press ENTER", container=False)
+with gr.Blocks(title='Mini-Gemini') as demo:
+    gr.Markdown(title_markdown)
+    # state = default_conversation.copy()
+    state = gr.State()
+
+    with gr.Row():
+        with gr.Column(scale=3):
+            imagebox = gr.Image(label="Input Image", type="filepath")
+            image_process_mode = gr.Radio(
+                ["Crop", "Resize", "Pad", "Default"],
+                value="Default",
+                label="Preprocess for non-square image", visible=False)
+
+            gr.Examples(examples=[
+                ["./minigemini/serve/examples/monday.jpg", "Explain why this meme is funny, and generate a picture when the weekend coming."],
+                ["./minigemini/serve/examples/woolen.png", "Show me one idea of what I could make with this?"],
+                ["./minigemini/serve/examples/extreme_ironing.jpg", "What is unusual about this image?"],
+                ["./minigemini/serve/examples/waterview.jpg", "What are the things I should be cautious about when I visit here?"],
+            ], inputs=[imagebox, textbox])
+
+            with gr.Accordion("Function", open=True) as parameter_row:
+                gen_image = gr.Radio(choices=['Yes', 'No'], value='No', interactive=True, label="Generate Image")
+
+            with gr.Accordion("Parameters", open=False) as parameter_row:
+                temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature",)
+                top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P",)
+                max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",)
+
+        with gr.Column(scale=7):
+            chatbot = gr.Chatbot(
+                elem_id="chatbot",
+                label="Mini-Gemini Chatbot",
+                height=850,
+                layout="panel",
+            )
+            with gr.Row():
+                with gr.Column(scale=7):
+                    textbox.render()
+                with gr.Column(scale=1, min_width=50):
+                    submit_btn = gr.Button(value="Send", variant="primary")
+            with gr.Row(elem_id="buttons") as button_row:
+                upvote_btn = gr.Button(value="👍  Upvote", interactive=False)
+                downvote_btn = gr.Button(value="👎  Downvote", interactive=False)
+                flag_btn = gr.Button(value="⚠️  Flag", interactive=False)
+                regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+                clear_btn = gr.Button(value="🗑️  Clear", interactive=False)
+
+    gr.Markdown(function_markdown)
+    gr.Markdown(tos_markdown)
+    gr.Markdown(learn_more_markdown)
+
+    btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]
+    upvote_btn.click(
+        upvote_last_response,
+        [state],
+        [textbox, upvote_btn, downvote_btn, flag_btn]
+    )
+    downvote_btn.click(
+        downvote_last_response,
+        [state],
+        [textbox, upvote_btn, downvote_btn, flag_btn]
+    )
+    flag_btn.click(
+        flag_last_response,
+        [state],
+        [textbox, upvote_btn, downvote_btn, flag_btn]
+    )
+    clear_btn.click(
+        clear_history,
+        None,
+        [state, chatbot, textbox, imagebox] + btn_list,
+        queue=False
+    )
+    regenerate_btn.click(
+        delete_text,
+        [state, image_process_mode],
+        [state, chatbot, textbox, imagebox] + btn_list,
+    ).then(
+        generate,
+        [state, imagebox, textbox, image_process_mode, gen_image, temperature, top_p, max_output_tokens],
+        [state, chatbot, textbox, imagebox] + btn_list,
+    )
+    textbox.submit(
+        add_text,
+        [state, imagebox, textbox, image_process_mode, gen_image],
+        [state, chatbot, textbox, imagebox] + btn_list,
+    ).then(
+        generate,
+        [state, imagebox, textbox, image_process_mode, gen_image, temperature, top_p, max_output_tokens],
+        [state, chatbot, textbox, imagebox] + btn_list,
+    )
+    submit_btn.click(
+        add_text,
+        [state, imagebox, textbox, image_process_mode, gen_image],
+        [state, chatbot, textbox, imagebox] + btn_list,
+    ).then(
+        generate,
+        [state, imagebox, textbox, image_process_mode, gen_image, temperature, top_p, max_output_tokens],
+        [state, chatbot, textbox, imagebox] + btn_list,
+    )
+
+
+demo.launch(debug=True)
+

requirements.txt

@@ -0,0 +1,9 @@
+transformers==4.36.2
+torch
+accelerate
+open_clip_torch
+diffusers
+deepspeed
+gradio==4.25.0
+gradio-client==0.15.0
+diffusers==0.26.3