Upload app.py

app.py

@@ -7,11 +7,13 @@ Original file is located at
     https://colab.research.google.com/drive/1goHcmXF0Gc4_X9PN-zecV77j9KeI6Dmn
 """
 
-#!pip install -q gradio
+# !pip install -q -U gradio
+# !pip install -q -U torch transformers accelerate einops
+# !pip install -q peft
+
+
 
-#!pip install torch transformers accelerate einops
 
-#!pip install peft
 
 import gradio as gr
 
@@ -22,7 +24,6 @@ from transformers import (
     TextIteratorStreamer,
     pipeline,
 )
-from threading import Thread
 
 # The huggingface model id for Microsoft's phi-2 model
 # Download and load model and tokenizer
@@ -32,8 +33,8 @@ tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2", trust_remote_code=T
 from peft import PeftModel, PeftConfig
 
 config = PeftConfig.from_pretrained("mudogruer/phi-2-SciQ")
-model = AutoModelForCausalLM.from_pretrained("microsoft/phi-2")
-model = PeftModel.from_pretrained(model, "mudogruer/phi-2-SciQ")
+base_model = AutoModelForCausalLM.from_pretrained("microsoft/phi-2")
+model = PeftModel.from_pretrained(base_model, "mudogruer/phi-2-SciQ")
 
 # Text generation pipeline
 phi2 = pipeline(
@@ -45,79 +46,22 @@ phi2 = pipeline(
     device_map="cpu",
 )
 
-
-# Function that accepts a prompt and generates text using the phi2 pipeline
-def generate(message, chat_history, max_new_tokens):
+def generate(message, max_new_tokens):
     instruction = "You are a helpful assistant to 'User'. You do not respond as 'User' or pretend to be 'User'. You only respond once as 'Assistant'."
-    final_prompt = f"Instruction: {instruction}\n"
-
-    for sent, received in chat_history:
-        final_prompt += "User: " + sent + "\n"
-        final_prompt += "Assistant: " + received + "\n"
-
-    final_prompt += "User: " + message + "\n"
-    final_prompt += "Output:"
-
-    # if (
-    #     len(tokenizer.tokenize(final_prompt))
-    #     >= tokenizer.model_max_length - max_new_tokens
-    # ):
-    #     final_prompt = "Instruction: Say 'Input exceeded context size, please clear the chat history and retry!' Output:"
-
-    # Streamer
-    streamer = TextIteratorStreamer(
-        tokenizer=tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=300.0
-    )
-    thread = Thread(
-        target=phi2,
-        kwargs={
-            "text_inputs": final_prompt,
-            "max_new_tokens": max_new_tokens,
-            "streamer": streamer,
-        },
-    )
-    thread.start()
-
-    generated_text = ""
-    for word in streamer:
-        generated_text += word
-        response = generated_text.strip()
-
-        if "User:" in response:
-            response = response.split("User:")[0].strip()
-
-        if "Assistant:" in response:
-            response = response.split("Assistant:")[1].strip()
-
-        yield response
-
-
-# Chat interface with gradio
+    final_prompt = f"Instruction: {instruction}\nUser: {message}\nOutput:"
+
+    # Generate text synchronously
+    response = phi2(final_prompt, max_new_tokens=max_new_tokens)
+    generated_text = response[0]['generated_text']
+
+    # Process to extract the last assistant's response
+    # Assuming the last line after 'Output:' is the response
+    last_response = generated_text.split('Output:')[-1].strip()
+    return last_response
+
+# Update the Gradio interface setup
 with gr.Blocks() as demo:
-    gr.Markdown(
-        """
-  # Phi-2 Scientific Question Chatbot
-  This chatbot was created using Microsoft's 2.7 billion parameter [phi-2](https://huggingface.co/microsoft/phi-2) Transformer model.
-
-  Phi-2 model was fine-tuned with questions including highschool level physics chemistry biology QA using SciQ dataset. In order to reduce the response time on this hardware, `max_new_tokens` has been set to `21` in the text generation pipeline. With this default configuration, it takes approximately `60 seconds` for the response to start being generated, and streamed one word at a time. Use the slider below to increase or decrease the length of the generated text.
-
-  For the safetensor: huggingface.co/mudogruer
-  """
-    )
-
-    tokens_slider = gr.Slider(
-        8,
-        128,
-        value=21,
-        label="Maximum new tokens",
-        info="A larger `max_new_tokens` parameter value gives you longer text responses but at the cost of a slower response time.",
-    )
-
-    chatbot = gr.ChatInterface(
-        fn=generate,
-        additional_inputs=[tokens_slider],
-        stop_btn=None,
-        examples=[["Who is Leonhard Euler?"]],
-    )
-
-demo.queue().launch()
+    gr.Markdown("""### Phi-2 Scientific Question Chatbot""")
+    tokens_slider = gr.Slider(8, 128, value=21, label="Maximum new tokens")
+    chatbot = gr.Interface(fn=generate, inputs=["text", tokens_slider], outputs="text")
+    demo.launch()