Create app.py

app.py

@@ -0,0 +1,79 @@
+import gradio as gr
+import base64
+import requests
+import io
+from PIL import Image
+import numpy as np
+
+API_KEY = os.environ['API_KEY']
+URL = os.environ['URL']
+
+def sketch_to_text(image):
+    if image is None or not isinstance(image, dict) or 'composite' not in image:
+        return "Please write something first."
+    
+    # Extract the image data from the dictionary
+    image_data = image['composite']
+    # Convert the image data to a PIL Image
+    pil_image = Image.fromarray(image_data.astype(np.uint8))
+    
+    # Convert the image to base64
+    buffered = io.BytesIO()
+    pil_image.save(buffered, format="PNG")
+    img_str = base64.b64encode(buffered.getvalue()).decode()
+
+    # Prepare the API request
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {API_KEY}"
+    }
+    payload = {
+        "model": "Llama-3.2-11B-Vision-Instruct",
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "Please read the forumla in the image and transcribe it into latex code, respond with code only, make sure the code is enclosed within a pair of $$"
+                    },
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/png;base64,{img_str}"
+                        }
+                    }
+                ]
+            }
+        ],
+        "max_tokens": 300
+    }
+
+    # Make the API request
+    response = requests.post(URL, headers=headers, json=payload)
+    
+    if response.status_code == 200:
+        return response.json()["choices"][0]["message"]["content"], response.json()["choices"][0]["message"]["content"]
+    else:
+        return f"Error: {response.status_code}, {response.text}", f"Error: {response.status_code}, {response.text}"
+
+# Create the Gradio interface
+with gr.Blocks() as iface:
+    gr.Markdown("# Pix2Latex")
+    gr.Markdown("Transcribing handwritten forumla into latex with Llama3.2 instruct")
+    
+    with gr.Column(scale=1):
+        input_image = gr.ImageEditor()
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            output1 = gr.Textbox(label="Raw")
+        with gr.Column(scale=1):
+            output2 = gr.Markdown(label="Rendered")
+    
+    input_image.change(fn=sketch_to_text, inputs=input_image, outputs=[output1, output2])
+    
+    gr.Markdown("How to use: 1. write your formula in the box above. 2. See it in real time, have fun doing math?")
+
+# Launch the app
+iface.launch()