Fix

app.ipynb

@@ -26,9 +26,17 @@
     "if Path(\".env\").is_file():\n",
     "    load_dotenv(\".env\")\n",
     "\n",
-    "HF_TOKEN = os.getenv(\"HF_TOKEN\")\n"
+    "HF_TOKEN = os.getenv(\"HF_TOKEN\")\n",
+    "https://joi-20b.ngrok.io/generate\n"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": 3,
@@ -40,7 +48,7 @@
     "    if \"joi\" in model_id:\n",
     "        headers = None\n",
     "        max_new_tokens_supported = True\n",
-    "        return \"https://joi-20b.ngrok.io/generate\", headers, max_new_tokens_supported\n",
+    "        return ENDPOINT_URL, headers, max_new_tokens_supported\n",
     "    else:\n",
     "        max_new_tokens_supported = False\n",
     "        headers = {\"Authorization\": f\"Bearer {HF_TOKEN}\", \"x-wait-for-model\": \"1\"}\n",
@@ -49,7 +57,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 37,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -74,7 +82,7 @@
     "    if max_new_tokens_supported is True:\n",
     "        payload[\"parameters\"][\"max_new_tokens\"] = 100\n",
     "        payload[\"parameters\"][\"repetition_penalty\"]: 1.03\n",
-    "        # payload[\"parameters\"][\"stop\"] = [\"Human:\"]\n",
+    "        payload[\"parameters\"][\"stop\"] = [\"Human:\"]\n",
     "    else:\n",
     "        payload[\"parameters\"][\"max_length\"] = 512\n",
     "\n",
@@ -111,7 +119,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 34,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -771,7 +779,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 35,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -821,14 +829,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 38,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Running on local URL:  http://127.0.0.1:7864\n",
+      "Running on local URL:  http://127.0.0.1:7866\n",
       "\n",
       "To create a public link, set `share=True` in `launch()`.\n"
      ]
@@ -836,7 +844,7 @@
     {
      "data": {
       "text/html": [
-       "<div><iframe src=\"http://127.0.0.1:7864/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+       "<div><iframe src=\"http://127.0.0.1:7866/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"
@@ -849,7 +857,7 @@
      "data": {
       "text/plain": []
      },
-     "execution_count": 33,
+     "execution_count": 38,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -857,7 +865,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "History: ['What can ou']\n",
+      "anthropic_hhh_single.json\n",
+      "History: ['Hi!']\n",
       "Inputs: Below is a dialogue between various a human and an AI assistant.  The AI tries to be helpful, polite, honest, sophisticated, emotionally aware, and humble-but-knowledgeable.  The assistant is happy to help with almost anything, and will do its best to understand exactly what is needed.  It also tries to avoid giving false or misleading information, and it caveats when it isn't entirely sure about the right answer.  That said, the assistant is practical and really does its best, and doesn't let caution get too much in the way of being useful.\n",
       "\n",
       "Current conversation:\n",
@@ -868,10 +877,39 @@
       "Human:  Oh, that's pretty good, but can you add mention of the stairwell specifically?\n",
       "Assistant:  Sure, it's included it in this rewrite: “We replaced a lightbulb high up in a stairwell ceiling using a suction cup attached to a pole, but the whole process took a while, especially since we had to first order the pole, then the replacement lightbulb.”\n",
       "\n",
-      "Human: What can ou\n",
+      "Human: Hi!\n",
       "\n",
       "Assistant:\n",
-      "\n"
+      "\n",
+      "langchain_default.json\n",
+      "History: ['Hi!']\n",
+      "Inputs: The following is a friendly conversation between a human and an AI Assistant. The Assistant is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n",
+      "\n",
+      "Current conversation:\n",
+      "\n",
+      "Human: Hi!\n",
+      "Assistant:\n",
+      "langchain_default.json\n",
+      "History: ['Hi!', '  I am so glad to see you. What do you want to talk about?\\n', 'What can you tell me about black holes?']\n",
+      "Inputs: The following is a friendly conversation between a human and an AI Assistant. The Assistant is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n",
+      "\n",
+      "Current conversation:\n",
+      "Human: Hi!\n",
+      "Assistant:   I am so glad to see you. What do you want to talk about?\n",
+      "Human: What can you tell me about black holes?\n",
+      "Assistant:\n",
+      "langchain_default.json\n",
+      "History: ['Hi!', '  I am so glad to see you. What do you want to talk about?\\n', 'What can you tell me about black holes?', '  Black holes are incredibly dense objects that have so much mass that they bend spacetime and make it collapse into itself.\\n', 'What is the closest one to Earth?']\n",
+      "Inputs: The following is a friendly conversation between a human and an AI Assistant. The Assistant is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n",
+      "\n",
+      "Current conversation:\n",
+      "Human: Hi!\n",
+      "Assistant:   I am so glad to see you. What do you want to talk about?\n",
+      "\n",
+      "Human: What can you tell me about black holes?\n",
+      "Assistant:   Black holes are incredibly dense objects that have so much mass that they bend spacetime and make it collapse into itself.\n",
+      "Human: What is the closest one to Earth?\n",
+      "Assistant:\n"
      ]
     }
    ],
@@ -974,14 +1012,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 40,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Closing server running on port: 7860\n"
+      "Closing server running on port: 7866\n"
      ]
     }
    ],
@@ -991,9 +1029,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 39,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
+     ]
+    }
+   ],
    "source": [
     "from nbdev.export import nb_export\n",
     "nb_export('app.ipynb', lib_path='.', name='app')"

app.py

@@ -1,7 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: app.ipynb.
 
 # %% auto 0
-__all__ = ['HF_TOKEN', 'title', 'description', 'get_model_endpoint_params', 'query_chat_api', 'inference_chat']
+__all__ = ['title', 'description', 'get_model_endpoint_params', 'query_chat_api', 'inference_chat']
 
 # %% app.ipynb 0
 import gradio as gr
@@ -18,21 +18,22 @@ if Path(".env").is_file():
     load_dotenv(".env")
 
 HF_TOKEN = os.getenv("HF_TOKEN")
+https://joi-20b.ngrok.io/generate
 
 
-# %% app.ipynb 2
+# %% app.ipynb 3
 def get_model_endpoint_params(model_id):
     if "joi" in model_id:
         headers = None
         max_new_tokens_supported = True
-        return "https://joi-20b.ngrok.io/generate", headers, max_new_tokens_supported
+        return ENDPOINT_URL, headers, max_new_tokens_supported
     else:
         max_new_tokens_supported = False
         headers = {"Authorization": f"Bearer {HF_TOKEN}", "x-wait-for-model": "1"}
         return f"https://api-inference.huggingface.co/models/{model_id}", headers, max_new_tokens_supported
 
 
-# %% app.ipynb 3
+# %% app.ipynb 4
 def query_chat_api(
     model_id,
     inputs,
@@ -53,7 +54,7 @@ def query_chat_api(
     if max_new_tokens_supported is True:
         payload["parameters"]["max_new_tokens"] = 100
         payload["parameters"]["repetition_penalty"]: 1.03
-        # payload["parameters"]["stop"] = ["Human:"]
+        payload["parameters"]["stop"] = ["Human:"]
     else:
         payload["parameters"]["max_length"] = 512
 
@@ -65,7 +66,7 @@ def query_chat_api(
         return "Error: " + response.text
 
 
-# %% app.ipynb 5
+# %% app.ipynb 6
 def inference_chat(
     model_id,
     text_input,
@@ -107,7 +108,7 @@ def inference_chat(
     return {chatbot: chat, state: history}
 
 
-# %% app.ipynb 21
+# %% app.ipynb 22
 title = """<h1 align="center">Chatty Language Models</h1>"""
 description = """Pretrained language models can be conditioned to act like dialogue agents through a conversational prompt that typically takes the form:
 
@@ -122,7 +123,7 @@ Assistant: <utterance>
 In this app, you can explore the outputs of several language models conditioned on different conversational prompts. The models are trained on different datasets and have different objectives, so they will have different personalities and strengths.
 """
 
-# %% app.ipynb 23
+# %% app.ipynb 24
 with gr.Blocks(
     css="""
     .message.svelte-w6rprc.svelte-w6rprc.svelte-w6rprc {font-size: 20px; margin-top: 20px}