Update app.py

app.py

@@ -1,4 +1,11 @@
 from threading import Thread
+import logging
+import time
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
+)
 
 import torch
 import gradio as gr
@@ -6,24 +13,39 @@ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, TextIteratorStrea
 
 model_id = "pszemraj/flan-t5-large-instruct-dolly_hhrlhf"
 torch_device = "cuda" if torch.cuda.is_available() else "cpu"
-print("Running on device:", torch_device)
-print("CPU threads:", torch.get_num_threads())
+logging.info("Running on device:", torch_device)
+logging.info("CPU threads:", torch.get_num_threads())
 
 
 if torch_device == "cuda":
-    model = AutoModelForSeq2SeqLM.from_pretrained(model_id, load_in_8bit=True, device_map="auto")
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        model_id, load_in_8bit=True, device_map="auto"
+    )
 else:
     model = AutoModelForSeq2SeqLM.from_pretrained(model_id)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 
 
-def run_generation(user_text, top_p, temperature, top_k, max_new_tokens, use_generation_config=False):
+def run_generation(
+    user_text,
+    top_p,
+    temperature,
+    top_k,
+    max_new_tokens,
+    no_repeat_ngram_size=4,
+    length_penalty=1.0,
+    repetition_penalty=1.1,
+    use_generation_config=False,
+):
+    st = time.perf_counter()
     # Get the model and tokenizer, and tokenize the user text.
     model_inputs = tokenizer([user_text], return_tensors="pt").to(torch_device)
 
     # Start generation on a separate thread, so that we don't block the UI. The text is pulled from the streamer
     # in the main thread. Adds timeout to the streamer to handle exceptions in the generation thread.
-    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
+    streamer = TextIteratorStreamer(
+        tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True
+    )
     generate_kwargs = dict(
         model_inputs,
         streamer=streamer,
@@ -32,7 +54,8 @@ def run_generation(user_text, top_p, temperature, top_k, max_new_tokens, use_gen
         num_beams=1,
         top_p=top_p,
         temperature=float(temperature),
-        top_k=top_k
+        top_k=top_k,
+        no_repeat_ngram_size=no_repeat_ngram_size,
     )
     t = Thread(target=model.generate, kwargs=generate_kwargs)
     t.start()
@@ -42,15 +65,18 @@ def run_generation(user_text, top_p, temperature, top_k, max_new_tokens, use_gen
     for new_text in streamer:
         model_output += new_text
         yield model_output
+    logging.info("Total rt:\t{rt} sec".format(rt=round(time.perf_counter() - st, 3)))
     return model_output
 
 
 def reset_textbox():
-    return gr.update(value='')
+    return gr.update(value="")
 
 
 with gr.Blocks() as demo:
-    duplicate_link = "https://huggingface.co/spaces/joaogante/transformers_streaming?duplicate=true"
+    duplicate_link = (
+        "https://huggingface.co/spaces/joaogante/transformers_streaming?duplicate=true"
+    )
     gr.Markdown(
         "# 🤗 Transformers 🔥Streaming🔥 on Gradio\n"
         "This demo showcases the use of the "
@@ -65,26 +91,54 @@ with gr.Blocks() as demo:
         with gr.Column(scale=4):
             user_text = gr.Textbox(
                 placeholder="Write an email about an alpaca that likes flan",
-                label="User input"
+                label="User input",
             )
             model_output = gr.Textbox(label="Model output", lines=10, interactive=False)
             button_submit = gr.Button(value="Submit")
 
         with gr.Column(scale=1):
             max_new_tokens = gr.Slider(
-                minimum=1, maximum=1000, value=250, step=1, interactive=True, label="Max New Tokens",
+                minimum=1,
+                maximum=1000,
+                value=250,
+                step=1,
+                interactive=True,
+                label="Max New Tokens",
             )
             top_p = gr.Slider(
-                minimum=0.05, maximum=1.0, value=0.95, step=0.05, interactive=True, label="Top-p (nucleus sampling)",
+                minimum=0.05,
+                maximum=1.0,
+                value=0.95,
+                step=0.05,
+                interactive=True,
+                label="Top-p (nucleus sampling)",
             )
             top_k = gr.Slider(
-                minimum=1, maximum=50, value=50, step=1, interactive=True, label="Top-k",
+                minimum=1,
+                maximum=50,
+                value=50,
+                step=1,
+                interactive=True,
+                label="Top-k",
             )
             temperature = gr.Slider(
-                minimum=0.1, maximum=5.0, value=0.8, step=0.1, interactive=True, label="Temperature",
+                minimum=0.1,
+                maximum=5.0,
+                value=0.8,
+                step=0.1,
+                interactive=True,
+                label="Temperature",
             )
 
-    user_text.submit(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output)
-    button_submit.click(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output)
+    user_text.submit(
+        run_generation,
+        [user_text, top_p, temperature, top_k, max_new_tokens],
+        model_output,
+    )
+    button_submit.click(
+        run_generation,
+        [user_text, top_p, temperature, top_k, max_new_tokens],
+        model_output,
+    )
 
-    demo.queue(max_size=32).launch(enable_queue=True)
+    demo.queue(max_size=32).launch(enable_queue=True)