Create app.py

app.py

@@ -0,0 +1,33 @@
+import os
+import gradio as gr
+import torch
+import numpy as np
+from transformers import pipeline
+
+import torch
+print(f"Is CUDA available: {torch.cuda.is_available()}")
+print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}")
+
+pipe_flan = pipeline("text2text-generation", model="philschmid/flan-t5-xxl-sharded-fp16", model_kwargs={"load_in_8bit":True, "device_map": "auto"})
+pipe_vanilla = pipeline("text2text-generation", model="t5-large", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})
+
+title = "Flan T5 and Vanilla T5"
+description = "This demo compares [T5-large](https://huggingface.co/t5-large) and [Flan-T5-XX-large](https://huggingface.co/google/flan-t5-xxl). Note that T5 expects a very specific format of the prompts, so the examples below are not necessarily the best prompts to compare."
+
+def inference(text):
+  output_flan = pipe_flan(text, max_length=100)[0]["generated_text"]
+  output_vanilla = pipe_vanilla(text, max_length=100)[0]["generated_text"]
+  return [output_flan, output_vanilla]
+
+io = gr.Interface(
+  inference,
+  gr.Textbox(lines=3),
+  outputs=[
+    gr.Textbox(lines=3, label="Flan T5"),
+    gr.Textbox(lines=3, label="T5")
+  ],
+  title=title,
+  description=description,
+  examples=examples
+)
+io.launch()