Spaces:
Sleeping
Sleeping
Delete app.py
Browse files
app.py
DELETED
@@ -1,169 +0,0 @@
|
|
1 |
-
import os
|
2 |
-
import gradio as gr
|
3 |
-
from io import BytesIO
|
4 |
-
from PIL import Image, ImageDraw, ImageFont, ImageColor
|
5 |
-
import json
|
6 |
-
from google import genai
|
7 |
-
from google.genai.types import GenerateContentConfig, GoogleSearch, Tool
|
8 |
-
|
9 |
-
# Initialize GenAI Client
|
10 |
-
API_KEY = os.getenv("GOOGLE_API_KEY") # Set your API key in the environment variables
|
11 |
-
client = genai.Client(api_key=API_KEY)
|
12 |
-
MODEL_ID = "gemini-2.0-flash-exp"
|
13 |
-
|
14 |
-
bounding_box_system_instructions = """
|
15 |
-
Identify objects in the image related to maternity or baby products.
|
16 |
-
Return bounding boxes as a JSON array with labels. Limit to 25 objects.
|
17 |
-
Label objects with clear and descriptive names, including unique characteristics.
|
18 |
-
"""
|
19 |
-
|
20 |
-
# Helper functions
|
21 |
-
def parse_json(json_output):
|
22 |
-
"""
|
23 |
-
Parse JSON output from the Gemini model.
|
24 |
-
"""
|
25 |
-
lines = json_output.splitlines()
|
26 |
-
for i, line in enumerate(lines):
|
27 |
-
if line == "```json":
|
28 |
-
json_output = "\n".join(lines[i + 1:])
|
29 |
-
json_output = json_output.split("```")[0]
|
30 |
-
break
|
31 |
-
return json_output
|
32 |
-
|
33 |
-
def plot_bounding_boxes(image, bounding_boxes):
|
34 |
-
"""
|
35 |
-
Draw bounding boxes with labels on the image.
|
36 |
-
"""
|
37 |
-
im = image.copy()
|
38 |
-
width, height = im.size
|
39 |
-
draw = ImageDraw.Draw(im)
|
40 |
-
colors = ['red', 'green', 'blue', 'yellow', 'orange', 'pink', 'purple', 'cyan'] + [
|
41 |
-
colorname for (colorname, _) in ImageColor.colormap.items()
|
42 |
-
]
|
43 |
-
font = ImageFont.load_default()
|
44 |
-
|
45 |
-
try:
|
46 |
-
bounding_boxes_json = json.loads(bounding_boxes)
|
47 |
-
for i, bounding_box in enumerate(bounding_boxes_json):
|
48 |
-
color = colors[i % len(colors)]
|
49 |
-
abs_y1 = int(bounding_box["box_2d"][0] / 1000 * height)
|
50 |
-
abs_x1 = int(bounding_box["box_2d"][1] / 1000 * width)
|
51 |
-
abs_y2 = int(bounding_box["box_2d"][2] / 1000 * height)
|
52 |
-
abs_x2 = int(bounding_box["box_2d"][3] / 1000 * width)
|
53 |
-
|
54 |
-
if abs_x1 > abs_x2:
|
55 |
-
abs_x1, abs_x2 = abs_x2, abs_x1
|
56 |
-
if abs_y1 > abs_y2:
|
57 |
-
abs_y1, abs_y2 = abs_y2, abs_y1
|
58 |
-
|
59 |
-
# Draw the box and label
|
60 |
-
draw.rectangle(((abs_x1, abs_y1), (abs_x2, abs_y2)), outline=color, width=4)
|
61 |
-
if "label" in bounding_box:
|
62 |
-
draw.text((abs_x1 + 8, abs_y1 + 6), bounding_box["label"], fill=color, font=font)
|
63 |
-
except Exception as e:
|
64 |
-
print(f"Error drawing bounding boxes: {e}")
|
65 |
-
|
66 |
-
return im
|
67 |
-
|
68 |
-
def predict_bounding_boxes(image, prompt):
|
69 |
-
"""
|
70 |
-
Generate bounding boxes for the input image.
|
71 |
-
"""
|
72 |
-
try:
|
73 |
-
# Resize image and prepare for input
|
74 |
-
image = image.resize((1024, int(1024 * image.height / image.width)))
|
75 |
-
buffered = BytesIO()
|
76 |
-
image.save(buffered, format="JPEG")
|
77 |
-
image_bytes = buffered.getvalue()
|
78 |
-
|
79 |
-
# Request bounding box predictions
|
80 |
-
response = client.models.generate_content(
|
81 |
-
model=MODEL_ID,
|
82 |
-
contents=[prompt, image_bytes],
|
83 |
-
config=GenerateContentConfig(system_instruction=bounding_box_system_instructions)
|
84 |
-
)
|
85 |
-
bounding_boxes = parse_json(response.text)
|
86 |
-
if not bounding_boxes:
|
87 |
-
raise ValueError("No bounding boxes returned.")
|
88 |
-
|
89 |
-
result_image = plot_bounding_boxes(image, bounding_boxes)
|
90 |
-
return result_image, bounding_boxes
|
91 |
-
except Exception as e:
|
92 |
-
return image, f"Error: {e}"
|
93 |
-
|
94 |
-
def google_search_query(question, grounding_image=None):
|
95 |
-
"""
|
96 |
-
Generate responses with Google search grounding.
|
97 |
-
"""
|
98 |
-
try:
|
99 |
-
# Configure Google Search grounding
|
100 |
-
tools = [Tool(google_search=GoogleSearch())]
|
101 |
-
contents = [question]
|
102 |
-
|
103 |
-
# If an image grounding is available, include it
|
104 |
-
if grounding_image:
|
105 |
-
contents.append(grounding_image)
|
106 |
-
|
107 |
-
response = client.models.generate_content(
|
108 |
-
model=MODEL_ID,
|
109 |
-
contents=contents,
|
110 |
-
config=GenerateContentConfig(tools=tools)
|
111 |
-
)
|
112 |
-
grounded_response = response.candidates[0].text
|
113 |
-
return grounded_response
|
114 |
-
except Exception as e:
|
115 |
-
return f"Error: {str(e)}"
|
116 |
-
|
117 |
-
# Gradio Interface
|
118 |
-
with gr.Blocks(theme=gr.themes.Glass(secondary_hue="blue")) as app:
|
119 |
-
gr.Markdown("# Maternity & Baby Product Assistance")
|
120 |
-
|
121 |
-
with gr.Tab("Image + Chat"):
|
122 |
-
gr.Markdown("### Upload an image and chat about the products identified in it.")
|
123 |
-
with gr.Row():
|
124 |
-
input_image = gr.Image(type="pil", label="Upload Image")
|
125 |
-
input_prompt = gr.Textbox(label="Describe what to identify", placeholder="Example: Identify baby products.")
|
126 |
-
submit_bbox = gr.Button("Generate Bounding Boxes")
|
127 |
-
with gr.Row():
|
128 |
-
output_image = gr.Image(type="pil", label="Output Image with Bounding Boxes")
|
129 |
-
output_json = gr.Textbox(label="Bounding Boxes JSON")
|
130 |
-
submit_bbox.click(
|
131 |
-
fn=predict_bounding_boxes,
|
132 |
-
inputs=[input_image, input_prompt],
|
133 |
-
outputs=[output_image, output_json]
|
134 |
-
)
|
135 |
-
gr.Markdown("### Chat about the products.")
|
136 |
-
with gr.Row():
|
137 |
-
image_chatbox = gr.Chatbot(label="Image-based Chat")
|
138 |
-
image_question = gr.Textbox(lines=2, label="Ask a Question about the Image")
|
139 |
-
submit_image_chat = gr.Button("Chat")
|
140 |
-
def chat_with_image(image_question, chat_log, output_json):
|
141 |
-
grounded_response = google_search_query(image_question, grounding_image=output_json)
|
142 |
-
chat_log.append(("You", image_question))
|
143 |
-
chat_log.append(("AI", grounded_response))
|
144 |
-
return chat_log
|
145 |
-
submit_image_chat.click(
|
146 |
-
fn=chat_with_image,
|
147 |
-
inputs=[image_question, image_chatbox, output_json],
|
148 |
-
outputs=[image_chatbox]
|
149 |
-
)
|
150 |
-
|
151 |
-
with gr.Tab("Chat Only"):
|
152 |
-
gr.Markdown("### Ask any questions about maternity or baby products.")
|
153 |
-
with gr.Row():
|
154 |
-
chatbot = gr.Chatbot(label="Chat Responses")
|
155 |
-
with gr.Row():
|
156 |
-
question_input = gr.Textbox(lines=2, label="Ask a Question", placeholder="Example: What are the best baby strollers?")
|
157 |
-
submit_chat = gr.Button("Chat")
|
158 |
-
def chat_without_image(question, chat_log):
|
159 |
-
grounded_response = google_search_query(question)
|
160 |
-
chat_log.append(("You", question))
|
161 |
-
chat_log.append(("AI", grounded_response))
|
162 |
-
return chat_log
|
163 |
-
submit_chat.click(
|
164 |
-
fn=chat_without_image,
|
165 |
-
inputs=[question_input, chatbot],
|
166 |
-
outputs=[chatbot]
|
167 |
-
)
|
168 |
-
|
169 |
-
app.launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|