Upload llmdolphin.py

llmdolphin.py

@@ -3,6 +3,7 @@ import gradio as gr
 from pathlib import Path
 import re
 import torch
+from typing import Any
 from huggingface_hub import hf_hub_download, HfApi
 from llama_cpp import Llama
 from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
@@ -75,6 +76,27 @@ llm_models = {
     "ChatWaifu_22B_v2.0_preview.Q4_K_S.gguf": ["mradermacher/ChatWaifu_22B_v2.0_preview-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_v1.4.Q5_K_M.gguf": ["mradermacher/ChatWaifu_v1.4-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_v1.3.1.Q4_K_M.gguf": ["mradermacher/ChatWaifu_v1.3.1-GGUF", MessagesFormatterType.MISTRAL],
+    "dolphin-2.6-mistral-7b-dpo-laser.Q4_K_S.gguf": ["mradermacher/dolphin-2.6-mistral-7b-dpo-laser-GGUF", MessagesFormatterType.MISTRAL],
+    "Flowable-Docs-Llama-3.1-8B.Q5_K_M.gguf": ["mradermacher/Flowable-Docs-Llama-3.1-8B-GGUF", MessagesFormatterType.LLAMA_3],
+    "slimorca-gemma2-9b-fft.Q4_K_M.gguf": ["mradermacher/slimorca-gemma2-9b-fft-GGUF", MessagesFormatterType.ALPACA],
+    "TQ2.5-14B-Sugarquill-v1-Q4_K_M.gguf": ["bartowski/TQ2.5-14B-Sugarquill-v1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "magnum-v3-9b-customgemma2.i1-Q4_K_M.gguf": ["mradermacher/magnum-v3-9b-customgemma2-i1-GGUF", MessagesFormatterType.ALPACA],
+    "Captain_BMO-12B.Q4_K_M.gguf": ["mradermacher/Captain_BMO-12B-GGUF", MessagesFormatterType.MISTRAL],
+    "LemonP-8B-Model_Stock.i1-Q5_K_M.gguf": ["mradermacher/LemonP-8B-Model_Stock-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "Ice0.31-08.11-RP.i1-Q5_K_M.gguf": ["mradermacher/Ice0.31-08.11-RP-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "EVA-Qwen2.5-14B-v0.2.i1-Q4_K_M.gguf": ["mradermacher/EVA-Qwen2.5-14B-v0.2-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "L3.1-Dark-Planet-SpinFire-Uncensored-8B-D_AU-Q4_k_m.gguf": ["DavidAU/L3.1-Dark-Planet-SpinFire-Uncensored-8B-GGUF", MessagesFormatterType.LLAMA_3],
+    "EndlessRP-v2-7B.Q5_K_M.gguf": ["mradermacher/EndlessRP-v2-7B-GGUF", MessagesFormatterType.MISTRAL],
+    "badger-lambda-0-llama-3-8b.i1-Q5_K_M.gguf": ["mradermacher/badger-lambda-0-llama-3-8b-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "L3.1-Artemis-e2-8B.i1-Q5_K_M.gguf": ["mradermacher/L3.1-Artemis-e2-8B-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "MN-12B-Inferor-v0.0.Q4_K_M.gguf": ["mradermacher/MN-12B-Inferor-v0.0-GGUF", MessagesFormatterType.MISTRAL],
+    "Eclipse-13B-dpo.i1-Q4_K_M.gguf": ["mradermacher/Eclipse-13B-dpo-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Epic_Fiction-8b-v0.1.Q5_K_M.gguf": ["mradermacher/Epic_Fiction-8b-v0.1-GGUF", MessagesFormatterType.MISTRAL],
+    "Llama-3-8B-StoryGenerator.i1-Q5_K_M.gguf": ["mradermacher/Llama-3-8B-StoryGenerator-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "badger-mu-llama-3-8b.i1-Q5_K_M.gguf": ["mradermacher/badger-mu-llama-3-8b-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "badger-writer-llama-3-8b.i1-Q5_K_M.gguf": ["mradermacher/badger-writer-llama-3-8b-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "Qwen-Rui-SE.Q5_K_M.gguf": ["mradermacher/Qwen-Rui-SE-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Valor-7B-v0.1.i1-Q4_K_M.gguf": ["mradermacher/Valor-7B-v0.1-i1-GGUF", MessagesFormatterType.MISTRAL],
     "Fraken-Maid-TW-Slerp.i1-Q5_K_M.gguf": ["mradermacher/Fraken-Maid-TW-Slerp-i1-GGUF", MessagesFormatterType.MISTRAL],
     "KunoichiLake-2x7b.Q4_K_M.gguf": ["mradermacher/KunoichiLake-2x7b-GGUF", MessagesFormatterType.MISTRAL],
     "Llama-3.1-SuperNova-8B-Lite_TIES_with_Base-Q4_K_M.gguf": ["bartowski/Llama-3.1-SuperNova-8B-Lite_TIES_with_Base-GGUF", MessagesFormatterType.LLAMA_3],
@@ -908,25 +930,35 @@ llm_formats = {
 llm_languages = ["English", "Japanese", "Chinese", "Korean", "Spanish", "Portuguese", "German", "French", "Finnish", "Russian"]
 llm_models_tupled_list = []
 default_llm_model_filename = list(llm_models.keys())[0]
-override_llm_format = None
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 
-def to_list(s):
+def to_list(s: str):
     return [x.strip() for x in s.split(",") if not s == ""]
 
 
-def list_uniq(l):
+def list_uniq(l: list):
     return sorted(set(l), key=l.index)
 
 
+def get_state(state: dict, key: str):
+    if key in state.keys(): return state[key]
+    else:
+        print(f"State '{key}' not found.")
+        return None
+
+
+def set_state(state: dict, key: str, value: Any):
+    state[key] = value
+
+
 @wrapt_timeout_decorator.timeout(dec_timeout=3.5)
-def to_list_ja(s):
+def to_list_ja(s: str):
     s = re.sub(r'[、。]', ',', s)
     return [x.strip() for x in s.split(",") if not s == ""]
 
 
-def is_japanese(s):
+def is_japanese(s: str):
     import unicodedata
     for ch in s:
         name = unicodedata.name(ch, "") 
@@ -964,7 +996,7 @@ def download_llm_models():
         llm_models_tupled_list.append((name, value))
 
 
-def download_llm_model(filename):
+def download_llm_model(filename: str):
     if not filename in llm_models.keys(): return default_llm_model_filename
     try:
         hf_hub_download(repo_id = llm_models[filename][0], filename = filename, local_dir = llm_models_dir)
@@ -975,7 +1007,7 @@ def download_llm_model(filename):
     return filename
 
 
-def get_dolphin_model_info(filename):
+def get_dolphin_model_info(filename: str):
     md = "None"
     items = llm_models.get(filename, None)
     if items:
@@ -983,20 +1015,18 @@ def get_dolphin_model_info(filename):
     return md
 
 
-def select_dolphin_model(filename, progress=gr.Progress(track_tqdm=True)):
-    global override_llm_format
-    override_llm_format = None
+def select_dolphin_model(filename: str, state: dict, progress=gr.Progress(track_tqdm=True)):
+    set_state(state, "override_llm_format", None)
     progress(0, desc="Loading model...")
     value = download_llm_model(filename)
     progress(1, desc="Model loaded.")
     md = get_dolphin_model_info(filename)
-    return gr.update(value=value, choices=get_dolphin_models()), gr.update(value=get_dolphin_model_format(value)), gr.update(value=md)
+    return gr.update(value=value, choices=get_dolphin_models()), gr.update(value=get_dolphin_model_format(value)), gr.update(value=md), state
 
 
-def select_dolphin_format(format_name):
-    global override_llm_format
-    override_llm_format = llm_formats[format_name]
-    return gr.update(value=format_name)
+def select_dolphin_format(format_name: str, state: dict):
+    set_state(state, "override_llm_format", llm_formats[format_name])
+    return gr.update(value=format_name), state
 
 
 download_llm_model(default_llm_model_filename)
@@ -1017,14 +1047,14 @@ def get_key_from_value(d, val):
     return None
 
 
-def get_dolphin_model_format(filename):
+def get_dolphin_model_format(filename: str):
     if not filename in llm_models.keys(): filename = default_llm_model_filename
     format = llm_models[filename][1]
     format_name = get_key_from_value(llm_formats, format)
     return format_name
 
 
-def add_dolphin_models(query, format_name):
+def add_dolphin_models(query: str, format_name: str):
     global llm_models
     api = HfApi()
     add_models = {}
@@ -1055,8 +1085,6 @@ def add_dolphin_models(query, format_name):
     return gr.update(choices=choices, value=choices[-1][1])
 
 
-dolphin_output_language = "English"
-dolphin_sysprompt_mode = "Default"
 dolphin_system_prompt = {"Default": r'''You are a helpful AI assistant to generate messages for AI that outputs an image when I enter a message.
 The message must have the following [Tags] generated in strict accordance with the following [Rules]:
 ```
@@ -1247,8 +1275,11 @@ Output should be enclosed in //GENBEGIN//:// and //://GENEND//. The text to be g
 "Chat with LLM": r"You are a helpful AI assistant. Respond in <LANGUAGE>."}
 
 
-def get_dolphin_sysprompt():
-    prompt = re.sub('<LANGUAGE>', dolphin_output_language, dolphin_system_prompt.get(dolphin_sysprompt_mode, ""))
+def get_dolphin_sysprompt(state: dict={}):
+    dolphin_sysprompt_mode = get_state(state, "dolphin_sysprompt_mode")
+    dolphin_output_language = get_state(state, "dolphin_output_language")
+    prompt = re.sub('<LANGUAGE>', dolphin_output_language if dolphin_output_language else llm_languages[0],
+                    dolphin_system_prompt.get(dolphin_sysprompt_mode, dolphin_system_prompt[list(dolphin_system_prompt.keys())[0]]))
     return prompt
 
 
@@ -1256,23 +1287,21 @@ def get_dolphin_sysprompt_mode():
     return list(dolphin_system_prompt.keys())
 
 
-def select_dolphin_sysprompt(key: str):
-    global dolphin_sysprompt_mode
-    if not key in dolphin_system_prompt.keys():
-        dolphin_sysprompt_mode = "Default"
-    else:
-        dolphin_sysprompt_mode = key
-    return gr.update(value=get_dolphin_sysprompt())
+def select_dolphin_sysprompt(key: str, state: dict):
+    dolphin_sysprompt_mode = get_state(state, "dolphin_sysprompt_mode")
+    if not key in dolphin_system_prompt.keys(): dolphin_sysprompt_mode = "Default"
+    else: dolphin_sysprompt_mode = key
+    set_state(state, "dolphin_sysprompt_mode", dolphin_sysprompt_mode)
+    return gr.update(value=get_dolphin_sysprompt(state)), state
 
 
 def get_dolphin_languages():
     return llm_languages
 
 
-def select_dolphin_language(lang: str):
-    global dolphin_output_language
-    dolphin_output_language = lang
-    return gr.update(value=get_dolphin_sysprompt())
+def select_dolphin_language(lang: str, state: dict):
+    set_state(state, "dolphin_output_language", lang)
+    return gr.update(value=get_dolphin_sysprompt(state)), state
 
 
 @wrapt_timeout_decorator.timeout(dec_timeout=5.0)
@@ -1293,11 +1322,12 @@ def dolphin_respond(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
     try:
         progress(0, desc="Processing...")
-
+        override_llm_format = get_state(state, "override_llm_format")
         if override_llm_format:
             chat_template = override_llm_format
         else:
@@ -1363,8 +1393,10 @@ def dolphin_respond(
 
 def dolphin_parse(
     history: list[tuple[str, str]],
+    state: dict,
 ):
     try:
+        dolphin_sysprompt_mode = get_state(state, "dolphin_sysprompt_mode")
         if dolphin_sysprompt_mode == "Chat with LLM" or not history or len(history) < 1:
             return "", gr.update(), gr.update()
         msg = history[-1][0]
@@ -1392,12 +1424,14 @@ def dolphin_respond_auto(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
     try:
         #if not is_japanese(message): return [(None, None)]
         progress(0, desc="Processing...")
 
+        override_llm_format = get_state(state, "override_llm_format")
         if override_llm_format:
             chat_template = override_llm_format
         else:
@@ -1465,9 +1499,11 @@ def dolphin_respond_auto(
 def dolphin_parse_simple(
     message: str,
     history: list[tuple[str, str]],
+    state: dict,
 ):
     try:
         #if not is_japanese(message): return message
+        dolphin_sysprompt_mode = get_state(state, "dolphin_sysprompt_mode")
         if dolphin_sysprompt_mode == "Chat with LLM" or not history or len(history) < 1: return message
         msg = history[-1][0]
         raw_prompt = get_raw_prompt(msg)
@@ -1499,8 +1535,10 @@ def respond_playground(
     top_p,
     top_k,
     repeat_penalty,
+    state,
 ):
     try:
+        override_llm_format = get_state(state, "override_llm_format")
         if override_llm_format:
             chat_template = override_llm_format
         else: