app-gpt.py

import logging
import gradio as gr
import os
from roboflow import Roboflow
from dotenv import load_dotenv
from openai import OpenAI
import tempfile
import numpy as np
from PIL import Image, ImageDraw
import base64

# Load environment variables
load_dotenv()

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# Initialize API Keys
roboflow_key = os.getenv("ROBOFLOW_API_KEY")
if not roboflow_key:
    raise ValueError("ROBOFLOW_API_KEY is missing. Please add it to the .env file.")

openai_key = os.getenv("OPENAI_API_KEY")
if not openai_key:
    raise ValueError("OPENAI_API_KEY is missing. Please add it to the .env file.")

# Initialize Roboflow and OpenAI clients
rf = Roboflow(api_key=roboflow_key)
project = rf.workspace("alat-pelindung-diri").project("nescafe-4base")
model = project.version(16).model  # Commented-out YOLO model usage, if necessary

client_openai = OpenAI(api_key=openai_key)

# Function to detect objects and estimate occluded objects
def detect_and_estimate_objects(image):
    try:
        # Save image to temporary file
        with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as temp_file:
            image.save(temp_file, format="JPEG")
            temp_file_path = temp_file.name

        logger.info("Image saved successfully for processing.")

        # Step 1: YOLO detection
        predictions = model.predict(temp_file_path, confidence=70, overlap=80).json()
        class_count = {}
        object_positions = []

        # Draw bounding boxes
        draw = ImageDraw.Draw(image)
        for prediction in predictions['predictions']:
            class_name = prediction['class']
            x, y, width, height = prediction['x'], prediction['y'], prediction['width'], prediction['height']

            # Calculate bounding box coordinates
            left = int(x - width / 2)
            top = int(y - height / 2)
            right = int(x + width / 2)
            bottom = int(y + height / 2)

            # Draw bounding box
            draw.rectangle([left, top, right, bottom], outline="red", width=4)

            # Count occurrences of detected classes
            class_count[class_name] = class_count.get(class_name, 0) + 1
            object_positions.append((left, top, right, bottom))

        logger.info(f"YOLO detected objects: {class_count}")

        # Step 2: Prepare base64 encoding for GPT-4
        # Encode image to Base64
        with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as temp_file:
            image.save(temp_file, format="JPEG")
            temp_file_path = temp_file.name

        with open(temp_file_path, "rb") as image_file:
            base64_image = base64.b64encode(image_file.read()).decode("utf-8")
            logger.info(f"Base64 encoding successful. Length: {len(base64_image)}")

        # Step 3: Use GPT-4 to estimate occluded objects
        response = client_openai.chat.completions.create(
            model="gpt-4o",
            messages=[
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "text",
                            "text": """Please count the number of cans of the following Nestlé products in the image, including those that are partially obstructed or hidden.
                            For partially visible or obstructed cans, please estimate their number based on visible clues and assume that they belong to the same product in front of them.
                            Please count accurately the number of cans of the following Nestlé products in the image:
                            - Nescafe Mocha
                            - Nescafe Latte
                            - Nescafe Original
                            - Bear Brand
                            - Nescafe Cappuccino
                            - Nescafe Ice Black
                            - Nescafe Coconut Latte
                            - Nescafe Caramel
                            Please note that some products may be partially visible or obstructed, but are still important to count. Products that are only partially visible or obstructed. Think of them as cans of the same product in front of them.
                            Please count the visible cans as well as the occluded ones. For partially hidden cans, assume they are the same product and estimate their presence based on the visible portion.

                            Provide your response in the format:
                            Nescafé Mocha: [number]
                            Nescafé Latte: [number]
                            Nescafé Original: [number]
                            Bear Brand: [number]
                            Nescafé Cappuccino: [number]
                            Nescafé Ice Black: [number]
                            Nescafé Coconut Latte: [number]
                            Nescafé Caramel: [number]
                            Total Nestlé Products: [Total number of Nestlé products]""",
                        },
                        {
                            "type": "image_url",
                            "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"},
                        },
                    ],
                }
            ],
        )
        gpt_estimation = response.choices[0].message.content.strip()
        print(response.choices[0].message.content)

        logger.info(f"GPT-4 estimation: {gpt_estimation}")

        # Step 4: Combine YOLO and GPT results (without YOLO part now)
        result_text = f"Results from GPT-4:\n{gpt_estimation}"

        # Step 5: Return the result text without the grid visualization
        output_path = "/tmp/prediction_result.jpg"
        image.save(output_path)

        logger.info("Processed image saved successfully.")

        # Cleanup
        os.remove(temp_file_path)

        return output_path, result_text

    except Exception as e:
        logger.error(f"Error during processing: {e}")
        return None, f"Error: {e}"

# Create Gradio interface
with gr.Blocks() as iface:
    gr.Markdown("### Object Detection and Counting with YOLO and GPT-4 Assistance")
    with gr.Row():
        input_image = gr.Image(type="pil", label="Upload Image")
        output_image = gr.Image(label="Processed Image")
        output_text = gr.Textbox(label="Results", interactive=False)

    detect_button = gr.Button("Process Image")
    detect_button.click(
        fn=detect_and_estimate_objects,
        inputs=[input_image],
        outputs=[output_image, output_text]
    )

iface.launch(debug=True)