app.py

import os
import gradio as gr
import torch
import numpy as np
from transformers import pipeline

examples = [['Q: Can Geoffrey Hinton have a conversation with George Washington? Give the rationale before answering.'],['Translate to German:  My name is Arthur'], ['Please answer the following question. What is the boiling point of Nitrogen?']] 

print(f"Is CUDA available: {torch.cuda.is_available()}")
print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}")

#pipe_biogpt = pipeline("text-generation", model="microsoft/BioGPT-Large", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})
pipe_flan_t5 = pipeline("text-generation", model="google/flan-t5-xxl", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})
#pipe_gpt2 = pipeline("text-generation", model="gpt2", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})
pipe_flan_ul2 = pipeline("text-generation", model="google/flan-ul2", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})
#pipe_galactica = pipeline("text-generation", model="facebook/galactica-1.3b", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})

title = "LLM vs LLM"
description = "**Disclaimer:** this demo was made for research purposes."

def inference(text):
  #output_biogpt = pipe_biogpt(text, max_length=100)[0]["generated_text"]
  output_flan_t5 = pipe_flan_t5(text, max_length=100)[0]["generated_text"]
  #output_gpt2 = pipe_gpt2(text, max_length=100)[0]["generated_text"]
  pipe_flan_ul2 = pipe_flan_t5(text, max_length=100)[0]["generated_text"]
  #output_galactica = pipe_galactica(text, max_length=100)[0]["generated_text"]
  return [
      #output_biogpt, 
      output_flan_t5,
      #output_gpt2,
      pipe_flan_ul2,
      #output_galactica
  ]

io = gr.Interface(
  inference,
  gr.Textbox(lines=3),
  outputs=[
    #gr.Textbox(lines=3, label="Microsoft: BioGPT-Large"),
    gr.Textbox(lines=3, label="Google: FLAN-T5-XXL"),
    #gr.Textbox(lines=3, label="GPT-2"),
    gr.Textbox(lines=3, label="Google: FLAN-UL2"),
    #gr.Textbox(lines=3, label="Facebook: Galactica 1.3B"),
  ],
  title=title,
  description=description,
  examples=examples
)
io.launch()