Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/safari_google.png filter=lfs diff=lfs merge=lfs -text

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

app.py

@@ -0,0 +1,247 @@
+import base64
+import json
+from datetime import datetime
+import gradio as gr
+import torch
+from PIL import Image, ImageDraw
+from qwen_vl_utils import process_vision_info
+from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
+import ast
+import os
+from datetime import datetime
+import numpy as np
+
+# Define constants
+DESCRIPTION = "[ShowUI-2B Demo](https://huggingface.co/showlab/ShowUI-2B)"
+_SYSTEM = "Based on the screenshot of the page, I give a text description and you give its corresponding location. The coordinate represents a clickable location [x, y] for an element, which is a relative coordinate on the screenshot, scaled from 0 to 1."
+MIN_PIXELS = 256 * 28 * 28
+MAX_PIXELS = 1344 * 28 * 28
+
+# Load the model
+model = Qwen2VLForConditionalGeneration.from_pretrained(
+    "/users/difei/siyuan/showui_demo/showui-2b",
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+)
+
+# Load the processor
+processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-2B-Instruct", min_pixels=MIN_PIXELS, max_pixels=MAX_PIXELS)
+
+# Helper functions
+def draw_point(image_input, point=None, radius=5):
+    """Draw a point on the image."""
+    if isinstance(image_input, str):
+        image = Image.open(image_input)
+    else:
+        image = Image.fromarray(np.uint8(image_input))
+
+    if point:
+        x, y = point[0] * image.width, point[1] * image.height
+        ImageDraw.Draw(image).ellipse((x - radius, y - radius, x + radius, y + radius), fill='red')
+    return image
+
+def array_to_image_path(image_array):
+    """Save the uploaded image and return its path."""
+    if image_array is None:
+        raise ValueError("No image provided. Please upload an image before submitting.")
+    img = Image.fromarray(np.uint8(image_array))
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"image_{timestamp}.png"
+    img.save(filename)
+    return os.path.abspath(filename)
+
+def run_showui(image, query):
+    """Main function for inference."""
+    image_path = array_to_image_path(image)
+
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": _SYSTEM},
+                {"type": "image", "image": image_path, "min_pixels": MIN_PIXELS, "max_pixels": MAX_PIXELS},
+                {"type": "text", "text": query}
+            ],
+        }
+    ]
+
+    # Prepare inputs for the model
+    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    image_inputs, video_inputs = process_vision_info(messages)
+    inputs = processor(
+        text=[text],
+        images=image_inputs,
+        videos=video_inputs,
+        padding=True,
+        return_tensors="pt"
+    )
+    inputs = inputs.to("cuda")
+
+    # Generate output
+    generated_ids = model.generate(**inputs, max_new_tokens=128)
+    generated_ids_trimmed = [
+        out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+    ]
+    output_text = processor.batch_decode(
+        generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+    )[0]
+
+    # Parse the output into coordinates
+    click_xy = ast.literal_eval(output_text)
+
+    # Draw the point on the image
+    result_image = draw_point(image_path, click_xy, radius=10)
+    return result_image, str(click_xy)
+
+# Function to record votes
+def record_vote(vote_type, image_path, query, action_generated):
+    """Record a vote in a JSON file."""
+    vote_data = {
+        "vote_type": vote_type,
+        "image_path": image_path,
+        "query": query,
+        "action_generated": action_generated,
+        "timestamp": datetime.now().isoformat()
+    }
+    with open("votes.json", "a") as f:
+        f.write(json.dumps(vote_data) + "\n")
+    return f"Your {vote_type} has been recorded. Thank you!"
+
+# Helper function to handle vote recording
+def handle_vote(vote_type, image_path, query, action_generated):
+    """Handle vote recording by using the consistent image path."""
+    if image_path is None:
+        return "No image uploaded. Please upload an image before voting."
+    return record_vote(vote_type, image_path, query, action_generated)
+
+# Load logo and encode to Base64
+with open("./assets/showui.png", "rb") as image_file:
+    base64_image = base64.b64encode(image_file.read()).decode("utf-8")
+
+
+# Define layout and UI
+def build_demo(embed_mode, concurrency_count=1):
+    with gr.Blocks(title="ShowUI-2B Demo", theme=gr.themes.Default()) as demo:
+        # State to store the consistent image path
+        state_image_path = gr.State(value=None)
+
+        if not embed_mode:
+            # Replace the original description with new content
+            gr.HTML(
+                f"""
+                <div style="display: flex; align-items: center; justify-content: center; margin-bottom: 20px;">
+                    <!-- Logo on the left -->
+                    <a href="https://github.com/showlab/ShowUI" target="_blank" style="margin-right: 20px;">
+                        <img src="data:image/png;base64,{base64_image}" alt="ShowUI Logo" style="width: auto; height: 66px;"/>
+                    </a>
+                    <!-- Links on the right -->
+                    <div style="display: flex; gap: 15px; font-size: 20px;">
+                        <a href="https://github.com/showlab/ShowUI" target="_blank">🏠[Project Homepage]</a>
+                        <a href="https://github.com/showlab/ShowUI" target="_blank">🤖[Code]</a>
+                        <a href="https://huggingface.co/showlab/ShowUI-2B" target="_blank">😊[Model]</a>
+                        <a href="https://arxiv.org/" target="_blank">📚[Paper]</a>
+                    </div>
+                </div>
+                """
+            )
+
+        with gr.Row():
+            with gr.Column(scale=3):
+                # Input components
+                imagebox = gr.Image(type="numpy", label="Input Screenshot")
+                textbox = gr.Textbox(
+                    show_label=True,
+                    placeholder="Enter a query (e.g., 'Click Nahant')",
+                    label="Query",
+                )
+                submit_btn = gr.Button(value="Submit", variant="primary")
+
+                # Placeholder examples
+                gr.Examples(
+                    examples=[
+                        ["./examples/safari_google.png", "Click on search bar."],
+                        ["./examples/apple_music.png", "Click on star."],
+                    ],
+                    inputs=[imagebox, textbox],
+                    examples_per_page=2
+                )
+
+            with gr.Column(scale=8):
+                # Output components
+                output_img = gr.Image(type="pil", label="Output Image")
+                output_coords = gr.Textbox(label="Clickable Coordinates")
+
+                # Buttons for voting, flagging, regenerating, and clearing
+                with gr.Row(elem_id="action-buttons", equal_height=True):
+                    vote_btn = gr.Button(value="👍 Vote", variant="secondary")
+                    downvote_btn = gr.Button(value="👎 Downvote", variant="secondary")
+                    flag_btn = gr.Button(value="🚩 Flag", variant="secondary")
+                    regenerate_btn = gr.Button(value="🔄 Regenerate", variant="secondary")
+                    clear_btn = gr.Button(value="🗑️ Clear", interactive=True)  # Combined Clear button
+
+            # Define button actions
+            def on_submit(image, query):
+                """Handle the submit button click."""
+                if image is None:
+                    raise ValueError("No image provided. Please upload an image before submitting.")
+                
+                # Generate consistent image path and store it in the state
+                image_path = array_to_image_path(image)
+                return run_showui(image, query) + (image_path,)
+
+            submit_btn.click(
+                on_submit,
+                [imagebox, textbox],
+                [output_img, output_coords, state_image_path],
+            )
+
+            clear_btn.click(
+                lambda: (None, None, None, None, None),
+                inputs=None,
+                outputs=[imagebox, textbox, output_img, output_coords, state_image_path],  # Clear all outputs
+                queue=False
+            )
+
+            regenerate_btn.click(
+                lambda image, query, state_image_path: run_showui(image, query),
+                [imagebox, textbox, state_image_path],
+                [output_img, output_coords],
+            )
+
+            # Record vote actions without feedback messages
+            vote_btn.click(
+                lambda image_path, query, action_generated: handle_vote(
+                    "upvote", image_path, query, action_generated
+                ),
+                inputs=[state_image_path, textbox, output_coords],
+                outputs=[],
+                queue=False
+            )
+
+            downvote_btn.click(
+                lambda image_path, query, action_generated: handle_vote(
+                    "downvote", image_path, query, action_generated
+                ),
+                inputs=[state_image_path, textbox, output_coords],
+                outputs=[],
+                queue=False
+            )
+
+            flag_btn.click(
+                lambda image_path, query, action_generated: handle_vote(
+                    "flag", image_path, query, action_generated
+                ),
+                inputs=[state_image_path, textbox, output_coords],
+                outputs=[],
+                queue=False
+            )
+
+    return demo
+# Launch the app
+if __name__ == "__main__":
+    demo = build_demo(embed_mode=False)
+    demo.queue(api_open=False).launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=True
+    )

app_legacy.py

@@ -0,0 +1,128 @@
+import base64
+import gradio as gr
+import torch
+from PIL import Image, ImageDraw
+from qwen_vl_utils import process_vision_info
+from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
+import ast
+import os
+from datetime import datetime
+import numpy as np
+
+# Function to draw a point on the image
+def draw_point(image_input, point=None, radius=5):
+    if isinstance(image_input, str):
+        image = Image.open(image_input)
+    else:
+        image = Image.fromarray(np.uint8(image_input))
+
+    if point:
+        x, y = point[0] * image.width, point[1] * image.height
+        ImageDraw.Draw(image).ellipse((x - radius, y - radius, x + radius, y + radius), fill='red')
+    return image
+
+# Function to save the uploaded image and return its path
+def array_to_image_path(image_array):
+    if image_array is None:
+        raise ValueError("No image provided. Please upload an image before submitting.")
+    img = Image.fromarray(np.uint8(image_array))
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"image_{timestamp}.png"
+    img.save(filename)
+    return os.path.abspath(filename)
+
+# Load the model
+model = Qwen2VLForConditionalGeneration.from_pretrained(
+    # "./showui-2b",
+    "/users/difei/siyuan/showui_demo/showui-2b",
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+    # verbose=True,
+)
+
+# Define minimum and maximum pixel thresholds
+min_pixels = 256 * 28 * 28
+max_pixels = 1344 * 28 * 28
+
+# Load the processor
+processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-2B-Instruct", min_pixels=min_pixels, max_pixels=max_pixels)
+
+# Hugging Face Space description
+DESCRIPTION = "[ShowUI-2B Demo](https://huggingface.co/showlab/ShowUI-2B)"
+
+# Define the system instruction
+_SYSTEM = "Based on the screenshot of the page, I give a text description and you give its corresponding location. The coordinate represents a clickable location [x, y] for an element, which is a relative coordinate on the screenshot, scaled from 0 to 1."
+
+# Define the main function for inference
+def run_showui(image, query):
+    image_path = array_to_image_path(image)
+
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": _SYSTEM},
+                {"type": "image", "image": image_path, "min_pixels": min_pixels, "max_pixels": max_pixels},
+                {"type": "text", "text": query}
+            ],
+        }
+    ]
+
+    # Prepare inputs for the model
+    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    image_inputs, video_inputs = process_vision_info(messages)
+    inputs = processor(
+        text=[text],
+        images=image_inputs,
+        videos=video_inputs,
+        padding=True,
+        return_tensors="pt"
+    )
+    inputs = inputs.to("cuda")
+
+    # Generate output
+    generated_ids = model.generate(**inputs, max_new_tokens=128)
+    generated_ids_trimmed = [
+        out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+    ]
+    output_text = processor.batch_decode(
+        generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+    )[0]
+
+    # Parse the output into coordinates
+    click_xy = ast.literal_eval(output_text)
+
+    # Draw the point on the image
+    result_image = draw_point(image_path, click_xy, radius=10)
+    return result_image, str(click_xy)
+
+with open("./assets/showui.png", "rb") as image_file:
+    base64_image = base64.b64encode(image_file.read()).decode("utf-8")
+
+# Gradio UI
+with gr.Blocks() as demo:
+    gr.HTML(
+        f"""
+        <div style="text-align: center; margin-bottom: 20px;">
+            <a href="https://github.com/showlab/ShowUI" target="_blank">
+                <img src="data:image/png;base64,{base64_image}" alt="ShowUI Logo" style="width: 200px; height: auto;"/>
+            </a>
+        </div>
+        """
+    )
+    
+    gr.Markdown(DESCRIPTION)
+    with gr.Tab(label="ShowUI-2B Input"):
+        with gr.Row():
+            with gr.Column():
+                input_img = gr.Image(label="Input Screenshot")
+                text_input = gr.Textbox(label="Query (e.g., 'Click Nahant')")
+                submit_btn = gr.Button(value="Submit")
+            with gr.Column():
+                output_img = gr.Image(label="Output Image")
+                output_coords = gr.Textbox(label="Clickable Coordinates")
+
+        submit_btn.click(run_showui, [input_img, text_input], [output_img, output_coords])
+
+demo.queue(api_open=False)
+demo.launch()

requirements.txt

@@ -0,0 +1 @@
+qwen-vl-utils==0.0.8

showui-2b/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

showui-2b/README.md

@@ -0,0 +1,188 @@
+---
+tags:
+- GUI agents
+- vision-language-action model
+- computer use
+---
+[Github](https://github.com/showlab/ShowUI/tree/main) | [Quick Start](https://huggingface.co/showlab/ShowUI-2B)
+<img src="examples/showui.png" alt="ShowUI" width="640">
+
+ShowUI is a lightweight (2B) vision-language-action model designed for GUI agents.
+[![demo](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Models-blue)](https://huggingface.co/showlab/ShowUI-2B)&nbsp;
+## ⭐ Quick Start
+
+1. Load model
+```python
+import ast
+import torch
+from PIL import Image, ImageDraw
+from qwen_vl_utils import process_vision_info
+from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor
+
+def draw_point(image_input, point=None, radius=5):
+    if isinstance(image_input, str):
+        image = Image.open(BytesIO(requests.get(image_input).content)) if image_input.startswith('http') else Image.open(image_input)
+    else:
+        image = image_input
+
+    if point:
+        x, y = point[0] * image.width, point[1] * image.height
+        ImageDraw.Draw(image).ellipse((x - radius, y - radius, x + radius, y + radius), fill='red')
+    display(image)
+    return
+
+model = Qwen2VLForConditionalGeneration.from_pretrained(
+    "showlab/ShowUI-2B",
+    torch_dtype=torch.bfloat16,
+    device_map="auto"
+)
+
+min_pixels = 256*28*28
+max_pixels = 1344*28*28
+
+processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-2B-Instruct", min_pixels=min_pixels, max_pixels=max_pixels)
+```
+
+2. **UI Grounding**
+```python
+img_url = 'examples/web_dbd7514b-9ca3-40cd-b09a-990f7b955da1.png'
+query = "Nahant"
+
+
+_SYSTEM = "Based on the screenshot of the page, I give a text description and you give its corresponding location. The coordinate represents a clickable location [x, y] for an element, which is a relative coordinate on the screenshot, scaled from 0 to 1."
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "text", "text": _SYSTEM},
+            {"type": "image", "image": img_url, "min_pixels": min_pixels, "max_pixels": max_pixels},
+            {"type": "text", "text": query}
+        ],
+    }
+]
+
+text = processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True,
+)
+image_inputs, video_inputs = process_vision_info(messages)
+inputs = processor(
+    text=[text],
+    images=image_inputs,
+    videos=video_inputs,
+    padding=True,
+    return_tensors="pt",
+)
+inputs = inputs.to("cuda")
+
+generated_ids = model.generate(**inputs, max_new_tokens=128)
+generated_ids_trimmed = [
+    out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+]
+output_text = processor.batch_decode(
+    generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)[0]
+
+click_xy = ast.literal_eval(output_text)
+# [0.73, 0.21]
+
+draw_point(img_url, click_xy, 10)
+```
+
+This will visualize the grounding results like (where the red points are [x,y])
+
+![download](https://github.com/user-attachments/assets/8fe2783d-05b6-44e6-a26c-8718d02b56cb)
+
+3. **UI Navigation**
+- Set up system prompt.
+```python
+_NAV_SYSTEM = """You are an assistant trained to navigate the {_APP} screen. 
+Given a task instruction, a screen observation, and an action history sequence, 
+output the next action and wait for the next observation. 
+Here is the action space:
+{_ACTION_SPACE}
+"""
+
+_NAV_FORMAT = """
+Format the action as a dictionary with the following keys:
+{'action': 'ACTION_TYPE', 'value': 'element', 'position': [x,y]}
+
+If value or position is not applicable, set it as `None`.
+Position might be [[x1,y1], [x2,y2]] if the action requires a start and end position.
+Position represents the relative coordinates on the screenshot and should be scaled to a range of 0-1.
+"""
+
+action_map = {
+'web': """
+1. `CLICK`: Click on an element, value is not applicable and the position [x,y] is required. 
+2. `INPUT`: Type a string into an element, value is a string to type and the position [x,y] is required. 
+3. `SELECT`: Select a value for an element, value is not applicable and the position [x,y] is required. 
+4. `HOVER`: Hover on an element, value is not applicable and the position [x,y] is required.
+5. `ANSWER`: Answer the question, value is the answer and the position is not applicable.
+6. `ENTER`: Enter operation, value and position are not applicable.
+7. `SCROLL`: Scroll the screen, value is the direction to scroll and the position is not applicable.
+8. `SELECT_TEXT`: Select some text content, value is not applicable and position [[x1,y1], [x2,y2]] is the start and end position of the select operation.
+9. `COPY`: Copy the text, value is the text to copy and the position is not applicable.
+""",
+
+'phone': """
+1. `INPUT`: Type a string into an element, value is not applicable and the position [x,y] is required. 
+2. `SWIPE`: Swipe the screen, value is not applicable and the position [[x1,y1], [x2,y2]] is the start and end position of the swipe operation.
+3. `TAP`: Tap on an element, value is not applicable and the position [x,y] is required.
+4. `ANSWER`: Answer the question, value is the status (e.g., 'task complete') and the position is not applicable.
+5. `ENTER`: Enter operation, value and position are not applicable.
+"""
+}
+
+_NAV_USER = """{system}
+Task: {task}
+Observation: <|image_1|>
+Action History: {action_history}
+What is the next action?
+"""
+```
+
+```python
+img_url = 'examples/chrome.png'
+split='web'
+system_prompt = _NAV_SYSTEM.format(_APP=split, _ACTION_SPACE=action_map[split])
+query = "Search the weather for the New York city."
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "text", "text": system_prompt},
+            {"type": "image", "image": img_url, "min_pixels": min_pixels, "max_pixels": max_pixels},
+            {"type": "text", "text": query}
+        ],
+    }
+]
+
+text = processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True,
+)
+image_inputs, video_inputs = process_vision_info(messages)
+inputs = processor(
+    text=[text],
+    images=image_inputs,
+    videos=video_inputs,
+    padding=True,
+    return_tensors="pt",
+)
+inputs = inputs.to("cuda")
+
+generated_ids = model.generate(**inputs, max_new_tokens=128)
+generated_ids_trimmed = [
+    out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+]
+output_text = processor.batch_decode(
+    generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)[0]
+
+print(output_text)
+# {'action': 'CLICK', 'value': None, 'position': [0.49, 0.42]},
+# {'action': 'INPUT', 'value': 'weather for New York city', 'position': [0.49, 0.42]},
+# {'action': 'ENTER', 'value': None, 'position': None}
+```
+
+![download](https://github.com/user-attachments/assets/624097ea-06f2-4c8f-83f6-b6b9ee439c0c)

showui-2b/added_tokens.json

@@ -0,0 +1,16 @@
+{
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

showui-2b/config.json

@@ -0,0 +1,48 @@
+{
+  "_name_or_path": "Qwen/Qwen2-VL-2B-Instruct",
+  "architectures": [
+    "Qwen2VLForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2_vl",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "type": "mrope"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 4096,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "hidden_size": 1536,
+    "in_chans": 3,
+    "model_type": "qwen2_vl",
+    "spatial_patch_size": 14
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 151936
+}

showui-2b/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "_attn_implementation": "eager",
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "temperature": 0.01,
+  "top_k": 1,
+  "top_p": 0.001,
+  "transformers_version": "4.45.0.dev0"
+}

showui-2b/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "Qwen2VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "max_pixels": 12845056,
+    "min_pixels": 3136
+  },
+  "temporal_patch_size": 2
+}

showui-2b/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68080df785764e98976eb9cc93a07c6c69cf8a6933738496e02aef55b53d2aa3
+size 4418202778

showui-2b/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

showui-2b/tokenizer_config.json

@@ -0,0 +1,144 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 4096,
+  "pad_token": null,
+  "padding_side": "right",
+  "processor_class": "Qwen2VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}