Update app.py

app.py

@@ -1,15 +1,13 @@
-import os
 import gradio as gr
-import torch
-import numpy as np
-from transformers import pipeline
+import requests
 
-import torch
-print(f"Is CUDA available: {torch.cuda.is_available()}")
-print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}")
+UL2_API_URL = "https://api-inference.huggingface.co/models/google/flan-ul2"
+FLAN_API_URL = "https://api-inference.huggingface.co/models/google/flan-t5-xxl"
+
+def query(payload, api_url):
+    response = requests.request("POST", api_url, json={"inputs":payload})
+    return response.json()
 
-pipe_flan = pipeline("text2text-generation", model="google/flan-ul2", model_kwargs={"load_in_8bit":True, "device_map": "auto"})
-pipe_vanilla = pipeline("text2text-generation", model="t5-large", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})
 
 examples = [
   ["Please answer to the following question. Who is going to be the next Ballon d'or?"],
@@ -27,20 +25,20 @@ A: Roger started with 5 balls. 2 cans of 3 tennis balls each is 6 tennis balls.
 Q: A juggler can juggle 16 balls. Half of the balls are golf balls, and half of the golf balls are blue. How many blue golf balls are there?"""]
 ]
 
-title = "Flan UL2 and Vanilla T5"
-description = "This demo compares [T5-large](https://huggingface.co/t5-large) and [Flan-UL2](https://huggingface.co/google/flan-ul2). Note that T5 expects a very specific format of the prompts, so the examples below are not necessarily the best prompts to compare. This demo is highly inspired from the ['I like Flan' demo](https://huggingface.co/spaces/osanseviero/i-like-flan)"
+title = "Flan UL2 vs Flan T5 XXL"
+description = "This demo compares [Flan-T5-xxl](https://huggingface.co/google/flan-t5-xxl) and [Flan-UL2](https://huggingface.co/google/flan-ul2). Learn more about these models in their model card!"
 
 def inference(text):
-  output_flan = pipe_flan(text, max_length=100)[0]["generated_text"]
-  output_vanilla = pipe_vanilla(text, max_length=100)[0]["generated_text"]
-  return [output_flan, output_vanilla]
+    output_ul2 = query(text, api_url=UL2_API_URL)[0]["generated_text"]
+    output_flan = query(text, api_url=FLAN_API_URL)[0]["generated_text"]
+    return [output_ul2, output_flan]
 
 io = gr.Interface(
   inference,
   gr.Textbox(lines=3),
   outputs=[
-    gr.Textbox(lines=3, label="Flan UL2"),
-    gr.Textbox(lines=3, label="T5 Large")
+    gr.Textbox(lines=3, label="Flan T5-UL2"),
+    gr.Textbox(lines=3, label="Flan T5-XXL")
   ],
   title=title,
   description=description,