Update app.py

app.py

@@ -1,3 +1,6 @@
+"""Run codes."""
+# pylint: disable=line-too-long, broad-exception-caught, invalid-name, missing-function-docstring, too-many-instance-attributes, missing-class-docstring
+# ruff: noqa: E501
 import os
 import platform
 import random
@@ -13,11 +16,131 @@ from ctransformers import AutoModelForCausalLM
 from dl_hf_model import dl_hf_model
 from loguru import logger
 
+filename_list = [
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q2_K.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_L.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_M.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_S.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_0.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_M.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_S.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_0.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_1.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_M.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_S.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q6_K.bin",
+    "Wizard-Vicuna-7B-Uncensored.ggmlv3.q8_0.bin",
+]
+
+URL = "https://huggingface.co/s3nh/mamba-gpt-3b-GGML/resolve/main/mamba-gpt-3b.ggmlv3.q8_0.bin"  # 4.05G
+
+_ = (
+    "golay" in platform.node()
+    or "okteto" in platform.node()
+    or Path("/kaggle").exists()
+    # or psutil.cpu_count(logical=False) < 4
+    or 1  # run 7b in hf
+)
+
+if _:
+    # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q2_K.bin"
+    url = "https://huggingface.co/s3nh/mamba-gpt-3b-GGML/resolve/main/mamba-gpt-3b.ggmlv3.q8_0.bin"  # 2.87G
+    # url = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q4_K_M.bin"  # 2.87G
+    # url = "https://huggingface.co/TheBloke/llama2_7b_chat_uncensored-GGML/blob/main/llama2_7b_chat_uncensored.ggmlv3.q4_K_M.bin"  # 4.08G
+
+
+prompt_template = """Below is an instruction that describes a task. Write a response that appropriately completes the request.
+
+### Instruction: {user_prompt}
+
+### Response:
+"""
+
+prompt_template = """System: You are a helpful,
+respectful and honest assistant. Always answer as
+helpfully as possible, while being safe.  Your answers
+should not include any harmful, unethical, racist,
+sexist, toxic, dangerous, or illegal content. Please
+ensure that your responses are socially unbiased and
+positive in nature. If a question does not make any
+sense, or is not factually coherent, explain why instead
+of answering something not correct. If you don't know
+the answer to a question, please don't share false
+information.
+User: {prompt}
+Assistant: """
+
+prompt_template = """System: You are a helpful assistant.
+User: {prompt}
+Assistant: """
+
+prompt_template = """Question: {question}
+Answer: Let's work this out in a step by step way to be sure we have the right answer."""
+
+prompt_template = """[INST] <>
+You are a helpful, respectful and honest assistant. Always answer as helpfully as possible assistant. Think step by step.
+<>
+
+What NFL team won the Super Bowl in the year Justin Bieber was born?
+[/INST]"""
+
+prompt_template = """[INST] <<SYS>>
+You are an unhelpful assistant. Always answer as helpfully as possible. Think step by step. <</SYS>>
+
+{question} [/INST]
+"""
+
+prompt_template = """[INST] <<SYS>>
+You are a helpful assistant.
+<</SYS>>
 
-LLM = AutoModelForCausalLM.from_pretrained("s3nh/mamba-gpt-3b-GGML/mamba-gpt-3b.ggmlv3.q8_0.bin", 
-                                          model_type="llama")
+{question} [/INST]
+"""
+
+prompt_template = """### HUMAN:
+{question}
+
+### RESPONSE:"""
+
+_ = [elm for elm in prompt_template.splitlines() if elm.strip()]
+stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
+
+logger.debug(f"{stop_string=} not used")
+
+_ = psutil.cpu_count(logical=False) - 1
+cpu_count: int = int(_) if _ else 1
+logger.debug(f"{cpu_count=}")
 
+LLM = None
 
+try:
+    model_loc, file_size = dl_hf_model(url)
+except Exception as exc_:
+    logger.error(exc_)
+    raise SystemExit(1) from exc_
+
+LLM = AutoModelForCausalLM.from_pretrained(
+    model_loc,
+    model_type="llama",
+    # threads=cpu_count,
+)
+
+logger.info(f"done load llm {model_loc=} {file_size=}G")
+
+os.environ["TZ"] = "Asia/Shanghai"
+try:
+    time.tzset()  # type: ignore # pylint: disable=no-member
+except Exception:
+    # Windows
+    logger.warning("Windows, cant run time.tzset()")
+
+_ = """
+ns = SimpleNamespace(
+    response="",
+    generator=(_ for _ in []),
+)
+# """
 
 @dataclass
 class GenerationConfig:
@@ -336,4 +459,4 @@ else:
 concurrency_count = 1
 logger.info(f"{concurrency_count=}")
 
-block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)
+block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)