Upload app.py

app.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+"""Untitled12.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1goHcmXF0Gc4_X9PN-zecV77j9KeI6Dmn
+"""
+
+#!pip install -q gradio
+
+#!pip install torch transformers accelerate einops
+
+#!pip install peft
+
+import gradio as gr
+
+import torch
+from transformers import (
+    AutoTokenizer,
+    AutoModelForCausalLM,
+    TextIteratorStreamer,
+    pipeline,
+)
+from threading import Thread
+
+# The huggingface model id for Microsoft's phi-2 model
+# Download and load model and tokenizer
+tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2", trust_remote_code=True)
+
+#Download safetensor of adapter of fine-tune Phi-2 model
+from peft import PeftModel, PeftConfig
+
+config = PeftConfig.from_pretrained("mudogruer/phi-2-SciQ")
+model = AutoModelForCausalLM.from_pretrained("microsoft/phi-2")
+model = PeftModel.from_pretrained(model, "mudogruer/phi-2-SciQ")
+
+# Text generation pipeline
+phi2 = pipeline(
+    "text-generation",
+    tokenizer=tokenizer,
+    model=model,
+    pad_token_id=tokenizer.eos_token_id,
+    eos_token_id=tokenizer.eos_token_id,
+    device_map="cpu",
+)
+
+
+# Function that accepts a prompt and generates text using the phi2 pipeline
+def generate(message, chat_history, max_new_tokens):
+    instruction = "You are a helpful assistant to 'User'. You do not respond as 'User' or pretend to be 'User'. You only respond once as 'Assistant'."
+    final_prompt = f"Instruction: {instruction}\n"
+
+    for sent, received in chat_history:
+        final_prompt += "User: " + sent + "\n"
+        final_prompt += "Assistant: " + received + "\n"
+
+    final_prompt += "User: " + message + "\n"
+    final_prompt += "Output:"
+
+    if (
+        len(tokenizer.tokenize(final_prompt))
+        >= tokenizer.model_max_length - max_new_tokens
+    ):
+        final_prompt = "Instruction: Say 'Input exceeded context size, please clear the chat history and retry!' Output:"
+
+    # Streamer
+    streamer = TextIteratorStreamer(
+        tokenizer=tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=300.0
+    )
+    thread = Thread(
+        target=phi2,
+        kwargs={
+            "text_inputs": final_prompt,
+            "max_new_tokens": max_new_tokens,
+            "streamer": streamer,
+        },
+    )
+    thread.start()
+
+    generated_text = ""
+    for word in streamer:
+        generated_text += word
+        response = generated_text.strip()
+
+        if "User:" in response:
+            response = response.split("User:")[0].strip()
+
+        if "Assistant:" in response:
+            response = response.split("Assistant:")[1].strip()
+
+        yield response
+
+
+# Chat interface with gradio
+with gr.Blocks() as demo:
+    gr.Markdown(
+        """
+  # Phi-2 Scientific Question Chatbot
+  This chatbot was created using Microsoft's 2.7 billion parameter [phi-2](https://huggingface.co/microsoft/phi-2) Transformer model.
+
+  Phi-2 model was fine-tuned with questions including physics chemistry biology QA using SciQ dataset. In order to reduce the response time on this hardware, `max_new_tokens` has been set to `21` in the text generation pipeline. With this default configuration, it takes approximately `60 seconds` for the response to start being generated, and streamed one word at a time. Use the slider below to increase or decrease the length of the generated text.
+
+  For the safetensor: huggingface.co/mudogruer
+  """
+    )
+
+    tokens_slider = gr.Slider(
+        8,
+        128,
+        value=21,
+        label="Maximum new tokens",
+        info="A larger `max_new_tokens` parameter value gives you longer text responses but at the cost of a slower response time.",
+    )
+
+    chatbot = gr.ChatInterface(
+        fn=generate,
+        additional_inputs=[tokens_slider],
+        stop_btn=None,
+        examples=[["Who is Leonhard Euler?"]],
+    )
+
+demo.queue().launch()