Upload folder using huggingface_hub

chatmodel.py

@@ -0,0 +1,89 @@
+from typing import Dict, List, Literal, TypedDict
+
+
+from models import Model
+from pybars import Compiler
+compiler = Compiler()
+
+class Turn(TypedDict):
+    role: Literal["user", "assistant", "system"]
+    content: str
+
+def chatmsg(message:str, role:Literal["user", "assistant", "system"]):
+    return {"role": role, "content": message}
+
+conversation=List[Turn]
+
+class ChatModel:
+    def __init__(self,model:Model,sysprompt:str):
+        self.setModel(model)
+        self.setSysPrompt(sysprompt)
+    def __call__(self, msg:str):
+        raise NotImplementedError
+    def getconversation(self) -> conversation:
+        raise NotImplementedError
+    def conversationend(self) -> bool:
+        raise NotImplementedError
+    def setconversation(self,conversation:conversation):
+        raise NotImplementedError
+    def setSysPrompt(self,sysprompt:str):
+        def _eq(this, a,b):
+            return a==b
+        self.sysprompt=compiler.compile(sysprompt)({
+            "model":self.name
+        },helpers={"eq":_eq})
+        print(self.name+" SystemPrompt:\n"+self.sysprompt)
+    def setModel(self,model:Model):
+        self.model=model
+
+class SwapChatModel(ChatModel):
+    def __init__(self,model:Model,sysprompt:str):
+        super().__init__(model,sysprompt)
+        self.conversation=[]
+    def __call__(self, msg:str):
+        if "End of conversation." in [i["content"] for i in self.conversation]:
+            return
+        self.conversation.append(chatmsg(msg,"assistant"))
+        prompt="".join([
+            self.model.start(),
+            self.model.conv([chatmsg(self.sysprompt,"system")]),
+            self.model.conv(self.conversation),self.model.starttok("user")
+            ])
+        ret=self.model(prompt, stop=[".","\n \n","?\n",".\n","tile|>","\n"],max_tokens=100)
+        comp=ret["choices"][0]["text"]
+        if("<|end" in comp):
+            self.conversation.append(chatmsg(comp.removesuffix("<|end"),"user"))
+            self.conversation.append(chatmsg("End of conversation.","user"))
+        else:
+            self.conversation.append(chatmsg(comp,"user"))
+    def getconversation(self) -> conversation:
+        return self.conversation
+    def conversationend(self) -> bool:
+        return "End of conversation." in [i["content"] for i in self.conversation]
+    def setconversation(self,conversation:conversation):
+        self.conversation=conversation
+SwapChatModel.name="SwapChat"
+
+
+class InquiryChatModel(SwapChatModel):
+    def __init__(self,model:Model,sysprompt:str):
+        super().__init__(model,sysprompt)
+    def inquire(self,msg):
+        prompt="".join([
+            self.model.start(),
+            self.model.conv([chatmsg(self.sysprompt,"system")]),
+            self.model.conv(self.conversation),
+            self.model.conv([chatmsg(msg,"assistant")]),
+            self.model.starttok("system"),
+            "Is this conversation complete(true/false)?\n"
+            ])
+        ret=self.model(prompt, stop=[".","\n \n","?\n",".\n","tile|>","\n"],max_tokens=10)
+        print("system prompt:",ret["choices"][0]["text"])
+        if "true" in ret["choices"][0]["text"].lower():
+            self.conversation.append(chatmsg(msg,"user"))
+            self.conversation.append(chatmsg("End of conversation.","user"))
+    def __call__(self, msg:str):
+        self.inquire(msg)
+        super().__call__(msg)
+InquiryChatModel.name="InquiryChat"
+models=[SwapChatModel,InquiryChatModel]

interactive_test.py

@@ -2,26 +2,39 @@ from typing import Any, Dict, List
 import gradio as gr
 from llama_cpp import Llama
 
+import chatmodel
 from models import Phi35,models
 
-syspropmt=r"""
+sysprompt=r"""
+{{! This comment will not show up in the output}}
 The User will make an inquiry to the assistant.
 Fullfill the users inquiry.
-The User will write a message with his closing thoughts and the keyword "<|endtile|>" if his inquiry is fullfilled.
+{{#if (eq model "SwapChat")}}
+The User will write a message with his closing thoughts and the keyword "<|endtile|>" if his inquiry is fulfilled.
+{{/if}}
 The User will never have more than one inquiry in one conversation.
 The User will never complete his own inquiry.
 The User will never be a assistant.
 The User keep his message short in one sentence.
+{{#if (eq model "SwapChat")}}
 All conversations will end with "<|endtile|>".
+{{/if}}
 After each User message is one assistant response.
 There can never be more than one assistant response in succession.
-
+{{#if (eq model "SwapChat")}}
 Example:
 User: What is the capital?
 Assistant: Could you please specify which capital you are referring to?
 User: The capital of France
 Assistant: The capital of France is Paris
 User: <|endtile|>
+{{else}}
+Example:
+User: What is the capital?
+Assistant: Could you please specify which capital you are referring to?
+User: The capital of France
+Assistant: The capital of France is Paris
+{{/if}}
 """.strip()
 
 conversations:List[Dict[str, Any]]=[
@@ -79,9 +92,11 @@ conversations:List[Dict[str, Any]]=[
 def chatmsg(message, role):
     return {"role": role, "content": message}
 
-currmodel=Phi35()
 
 
+currmodel=Phi35()
+chat:chatmodel.ChatModel=chatmodel.models[0](currmodel,sysprompt)
+
 with gr.Blocks() as demo:
     with gr.Accordion("Info"):
         gr.Markdown(f"""
@@ -117,8 +132,15 @@ with gr.Blocks() as demo:
             return "", next(conversation for conversation in conversations if conversation["name"] == choice)["content"]
         convchoicebox.change(update_choicebox, [convchoicebox,custom_conv], [msg,chatbot])
 
+        msysprompt=gr.Textbox(value=sysprompt, label="System Prompt")
 
-        sysprompt=gr.Textbox(value=syspropmt, label="System Prompt")
+        def update_sysprompt(csysprompt:str):
+            global sysprompt
+            sysprompt=csysprompt
+            chat.setSysPrompt(sysprompt)
+            chat.setconversation([])
+            return "", chat.getconversation()
+        msysprompt.submit(update_sysprompt, [msysprompt], [msg,chatbot])
 
         #Choose Models
         modelchoicebox = gr.Radio(choices=[model.modelname for model in models], value=currmodel.modelname, label="Model")
@@ -126,25 +148,25 @@ with gr.Blocks() as demo:
             global currmodel
             currmodel.close()
             currmodel=next(model for model in models if model.modelname == choice)()
-            return "", []
+            chat.setModel(currmodel)
+            chat.setconversation([])
+            return "", chat.getconversation()
         modelchoicebox.change(update_modelchoicebox, [modelchoicebox], [msg,chatbot])
 
-    #generate response
-    def respond(message:str, chat_history:List[Dict[str, str]],syspropmt:str):
-        global currmodel
-        if "End of conversation." in [i["content"] for i in chat_history]:
-            return "", chat_history
-        chat_history.append(chatmsg(message,"assistant"))
+        chatchoicebox = gr.Radio(choices=[model.name for model in chatmodel.models], value=chat.name, label="Chat")
+        def update_chatchoicebox(choice):
+            global chat, currmodel, sysprompt
+            chat=next(model for model in chatmodel.models if model.name == choice)(currmodel,sysprompt)
+            chat.setconversation([])
+            return "", chat.getconversation()
+        chatchoicebox.change(update_chatchoicebox, [chatchoicebox], [msg,chatbot])
 
-        ret=currmodel(currmodel.conv([chatmsg(syspropmt,"system")])+currmodel.conv(chat_history)+currmodel.starttok("user"), stop=[".","\n \n","?\n",".\n","tile|>"],max_tokens=100)
-        comp=ret["choices"][0]["text"]
-        print(repr(comp))
-        if("<|end" in comp):
-            chat_history.append(chatmsg(comp.removesuffix("<|end"),"user"))
-            chat_history.append(chatmsg("End of conversation.","user"))
-        else:
-            chat_history.append(chatmsg(comp,"user"))
-        return "", chat_history
-    submit.click(respond, [msg, chatbot,sysprompt], [msg, chatbot])
-    msg.submit(respond, [msg, chatbot,sysprompt], [msg, chatbot])
-demo.launch()
+    #generate response
+    def respond(message:str,chatbot:List[Dict[str, str]]):
+        global chat
+        chat.setconversation(chatbot)
+        chat(message)
+        return "", chat.getconversation()
+    submit.click(respond, [msg,chatbot], [msg, chatbot])
+    msg.submit(respond, [msg,chatbot], [msg, chatbot])
+demo.launch()

models.py

@@ -1,17 +1,19 @@
 from typing import Dict, List
 
 from llama_cpp import Llama
-
+llama_args={"n_gpu_layers":100,"main_gpu":0,"verbose":True}
 
 class Model:
     def __init__(self):
         pass
     def __call__(self, msg:str, stop:List[str], max_tokens:int):
         raise NotImplementedError
-    def conv(self, msgs:List[Dict[str, str]]):
+    def conv(self, msgs:List[Dict[str, str]])->str:
         raise NotImplementedError
-    def starttok(self, user:str):
+    def starttok(self, user:str)->str:
         raise NotImplementedError
+    def start(self)->str:
+        return ""
     def close(self):
         pass
 
@@ -20,10 +22,13 @@ class Phi35RPMax(Model):
         self.llm = Llama.from_pretrained(
             repo_id="ArliAI/Phi-3.5-mini-3.8B-ArliAI-RPMax-v1.1-GGUF",
             filename="ArliAI-RPMax-3.8B-v1.1-fp16.gguf",
+            **llama_args,
+
         )
         
     def __call__(self, msg:str, stop:List[str], max_tokens:int):
-        return self.llm(msg, stop=stop, max_tokens=max_tokens)
+        ret=self.llm(msg, stop=stop, max_tokens=max_tokens)
+        return ret
     
     def conv(self,msgs:List[Dict[str, str]]):
         return "\n".join([f"<|{msg['role']}|>\n{msg['content']}<|end|>" for msg in msgs])
@@ -36,11 +41,12 @@ class Phi35(Model):
     def __init__(self):
         self.llm = Llama.from_pretrained(
             repo_id="bartowski/Phi-3.5-mini-instruct-GGUF",
-            filename="Phi-3.5-mini-instruct-IQ3_XS.gguf",
+            filename="Phi-3.5-mini-instruct-f32.gguf",
+            **llama_args,
         )
     def __call__(self, msg:str, stop:List[str], max_tokens:int):
         return self.llm(msg, stop=stop, max_tokens=max_tokens)
-    
+
     def conv(self,msgs:List[Dict[str, str]]):
         return "\n".join([f"<|{msg['role']}|>\n{msg['content']}<|end|>" for msg in msgs])
     
@@ -81,14 +87,18 @@ class Llama31uncensored(Model):
         self.llm = Llama.from_pretrained(
             repo_id="Orenguteng/Llama-3.1-8B-Lexi-Uncensored-V2-GGUF",
             filename="Llama-3.1-8B-Lexi-Uncensored_V2_F16.gguf",
+            **llama_args,
         )
     def __call__(self, msg:str, stop:List[str], max_tokens:int):
         return self.llm(msg, stop=stop, max_tokens=max_tokens)
     
+    def start(self):
+        return "<|begin_of_text|>"
+    
     def conv(self,msgs:List[Dict[str, str]]):
-        return "\n".join([f"<|begin_of_text|><|start_header_id|>{msg['role']}<|end_header_id|>\n\n{msg['content']}<|eot_id|>" for msg in msgs])
+        return "\n".join([f"<|start_header_id|>{msg['role']}<|end_header_id|>\n\n{msg['content']}<|eot_id|>" for msg in msgs])
     def starttok(self,user:str):
-        return f"<|begin_of_text|><|start_header_id|>{user}<|end_header_id|>\n\n"
+        return f"<|start_header_id|>{user}<|end_header_id|>\n\n"
     def close(self):
         self.llm.close()
 Llama31uncensored.modelname="Llama31-uncensored-fp16"
@@ -98,6 +108,7 @@ class Llama31(Model):
         self.llm = Llama.from_pretrained(
             repo_id="lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF",
             filename="Meta-Llama-3.1-8B-Instruct-IQ4_XS.gguf",
+            **llama_args,
         )
     def __call__(self, msg:str, stop:List[str], max_tokens:int):
         return self.llm(msg, stop=stop, max_tokens=max_tokens)

requirements.txt

@@ -1 +1,60 @@
-llama-cpp-python
+aiofiles==23.2.1
+annotated-types==0.7.0
+anyio==4.6.2.post1
+certifi==2024.8.30
+charset-normalizer==3.4.0
+click==8.1.7
+colorama==0.4.6
+contourpy==1.3.0
+cycler==0.12.1
+diskcache==5.6.3
+fastapi==0.112.4
+ffmpy==0.4.0
+filelock==3.16.1
+fonttools==4.54.1
+fsspec==2024.9.0
+gradio==4.43.0
+gradio_client==1.3.0
+h11==0.14.0
+httpcore==1.0.6
+httpx==0.27.2
+huggingface-hub==0.26.0
+idna==3.10
+importlib_resources==6.4.5
+Jinja2==3.1.4
+kiwisolver==1.4.7
+llama_cpp_python==0.2.90
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.9.2
+mdurl==0.1.2
+numpy==2.1.2
+orjson==3.10.7
+packaging==24.1
+pandas==2.2.3
+pillow==10.4.0
+pydantic==2.9.2
+pydantic_core==2.23.4
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.2.0
+python-dateutil==2.9.0.post0
+python-multipart==0.0.12
+pytz==2024.2
+PyYAML==6.0.2
+requests==2.32.3
+rich==13.9.2
+ruff==0.7.0
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.38.6
+tomlkit==0.12.0
+tqdm==4.66.5
+typer==0.12.5
+typing_extensions==4.12.2
+tzdata==2024.2
+urllib3==2.2.3
+uvicorn==0.32.0
+websockets==12.0