Upload folder using huggingface_hub

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Martin Thissen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,114 @@
 ---
-title: Llama2 Local
-emoji: 🌍
-colorFrom: blue
-colorTo: blue
+title: llama2_local
+app_file: llama.py
 sdk: gradio
-sdk_version: 3.41.2
-app_file: app.py
-pinned: false
+sdk_version: 3.37.0
 ---
+# Llama2 on Your Local Computer
+Run the new Llama2 and Llama2-Chat models on your local computer.
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Getting Started
+
+### Installation
+
+1. Clone the repository:
+```
+git clone https://github.com/thisserand/llama2_local.git
+cd llama2_local
+```
+
+2. Install required dependencies:
+```
+pip install -r requirements.txt
+```
+
+### Prerequisites
+To be able to download the model weights and tokenizer from Huggingface, you firtst need to visit the [Meta AI website](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) and accept their License (my request got approved within 30 minutes). Make sure that you state the email address that you are also using for your Huggingface account. Once your request got accepted, you need to go to one of the Llama2 Huggingface repositories (e.g., the Llama2-7B model) and request access for there again, as can be seen in the following image (access should be granted right away):
+![Huggingface Llama2 Access](./images/huggingface_llama2_access.png)
+
+Once you are all set with your access requests the last step is to login to your huggingface account in your current runtime. For this we will use the following command:
+```
+huggingface-cli login
+```
+You can find your Access Token here (https://huggingface.co/settings/tokens):
+![Huggingface Access Token](./images/huggingface_access_token.png)
+
+### Windows
+Make sure that you have gcc with version >=11 installed on your computer. Here are steps described by [Kevin Anthony Kaw
+](https://github.com/kevinkaw) for a successful setup of gcc:
+- CMake version cmake-3.27.0-windows-x86_64.msi installed to root directory ("C:")
+- minGW64 version 11.0.0 extracted to root directory ("C:")
+- set environment path variables for CMake and minGW64
+- install visual studio build tools. It's way at the bottom under "Tools for Visual Studio" drop down list.
+- In visual studio, check the "Desktop development with c++", click install.
+
+## Usage
+
+### Full Precision (Original)
+
+Llama2-7B:
+```
+python llama.py --model_name="meta-llama/Llama-2-7b-hf"
+```
+Llama2-7B-Chat:
+```
+python llama.py --model_name="meta-llama/Llama-2-7b-chat-hf"
+```
+Llama2-13B:
+```
+python llama.py --model_name="meta-llama/Llama-2-13b-hf"
+```
+Llama2-13B-Chat:
+```
+python llama.py --model_name="meta-llama/Llama-2-13b-chat-hf"
+```
+Llama2-70B:
+```
+python llama.py --model_name="meta-llama/Llama-2-70b-hf"
+```
+Llama2-70B-Chat:
+```
+python llama.py --model_name="meta-llama/Llama-2-70b-chat-hf"
+```
+### GPTQ Quantized
+Llama2-7B:
+```
+python llama.py --model_name="TheBloke/Llama-2-7B-GPTQ"
+```
+Llama2-7B-Chat:
+```
+python llama.py --model_name="TheBloke/Llama-2-7b-Chat-GPTQ"
+```
+Llama2-13B:
+```
+python llama.py --model_name="TheBloke/Llama-2-13B-GPTQ"
+```
+Llama2-13B-Chat:
+```
+python llama.py --model_name="TheBloke/Llama-2-13B-Chat-GPTQ"
+```
+Llama2-70B:
+```
+python llama.py --model_name="TheBloke/Llama-2-70B-GPTQ"
+```
+Llama2-70B-Chat:
+```
+python llama.py --model_name="TheBloke/Llama-2-70B-Chat-GPTQ"
+```
+### GGML Quantized
+Llama2-7B:
+```
+python llama.py --model_name="TheBloke/Llama-2-7B-GGML" --file_name="llama-2-7b.ggmlv3.q4_K_M.bin"
+```
+Llama2-7B-Chat:
+```
+python llama.py --model_name="TheBloke/Llama-2-7B-Chat-GGML" --file_name="llama-2-7b-chat.ggmlv3.q4_K_M.bin"
+```
+Llama2-13B:
+```
+python llama.py --model_name="TheBloke/Llama-2-13B-GGML" --file_name="llama-2-13b.ggmlv3.q4_K_M.bin"
+```
+Llama2-13B-Chat:
+```
+python llama.py --model_name="TheBloke/Llama-2-13B-Chat-GGML" --file_name="llama-2-13b-chat.ggmlv3.q4_K_M.bin"
+```

llama.py

@@ -0,0 +1,114 @@
+import os
+import gradio as gr
+import fire
+from enum import Enum
+from threading import Thread
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from auto_gptq import AutoGPTQForCausalLM
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
+from transformers import TextIteratorStreamer
+from llama_chat_format import format_to_llama_chat_style
+
+
+# class syntax
+class Model_Type(Enum):
+    gptq = 1
+    ggml = 2
+    full_precision = 3
+
+
+def get_model_type(model_name):
+  if "gptq" in model_name.lower():
+    return Model_Type.gptq
+  elif "ggml" in model_name.lower():
+    return Model_Type.ggml
+  else:
+    return Model_Type.full_precision
+
+
+def create_folder_if_not_exists(folder_path):
+    if not os.path.exists(folder_path):
+        os.makedirs(folder_path)
+
+
+def initialize_gpu_model_and_tokenizer(model_name, model_type):
+    if model_type == Model_Type.gptq:
+      model = AutoGPTQForCausalLM.from_quantized(model_name, device_map="auto", use_safetensors=True, use_triton=False)
+      tokenizer = AutoTokenizer.from_pretrained(model_name)
+    else:
+      model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", token=True)
+      tokenizer = AutoTokenizer.from_pretrained(model_name, token=True)
+    return model, tokenizer
+
+
+def init_auto_model_and_tokenizer(model_name, model_type, file_name=None):
+  model_type = get_model_type(model_name)
+
+  if Model_Type.ggml == model_type:
+    models_folder = "./models"
+    create_folder_if_not_exists(models_folder)
+    file_path = hf_hub_download(repo_id=model_name, filename=file_name, local_dir=models_folder)
+    model = Llama(file_path, n_ctx=4096)
+    tokenizer = None
+  else:
+    model, tokenizer = initialize_gpu_model_and_tokenizer(model_name, model_type=model_type)
+  return model, tokenizer
+
+
+def run_ui(model, tokenizer, is_chat_model, model_type):
+  with gr.Blocks() as demo:
+      chatbot = gr.Chatbot()
+      msg = gr.Textbox()
+      clear = gr.Button("Clear")
+
+      def user(user_message, history):
+          return "", history + [[user_message, None]]
+
+      def bot(history):
+          if is_chat_model:
+              instruction = format_to_llama_chat_style(history)
+          else:
+              instruction =  history[-1][0]
+
+          history[-1][1] = ""
+          kwargs = dict(temperature=0.6, top_p=0.9)
+          if model_type == Model_Type.ggml:
+              kwargs["max_tokens"] = 512
+              for chunk in model(prompt=instruction, stream=True, **kwargs):
+                  token = chunk["choices"][0]["text"]
+                  history[-1][1] += token
+                  yield history
+
+          else:
+              streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, Timeout=5)
+              inputs = tokenizer(instruction, return_tensors="pt").to(model.device)
+              kwargs["max_new_tokens"] = 512
+              kwargs["input_ids"] = inputs["input_ids"]
+              kwargs["streamer"] = streamer
+              thread = Thread(target=model.generate, kwargs=kwargs)
+              thread.start()
+
+              for token in streamer:
+                  history[-1][1] += token
+                  yield history
+
+      msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(bot, chatbot, chatbot)
+      clear.click(lambda: None, None, chatbot, queue=False)
+  demo.queue()
+  demo.launch(share=True, debug=True)
+
+def main(model_name=None, file_name=None):
+    assert model_name is not None, "model_name argument is missing."
+
+    is_chat_model = 'chat' in model_name.lower()
+    model_type = get_model_type(model_name)
+
+    if model_type == Model_Type.ggml:
+      assert file_name is not None, "When model_name is provided for a GGML quantized model, file_name argument must also be provided."
+
+    model, tokenizer = init_auto_model_and_tokenizer(model_name, model_type, file_name)
+    run_ui(model, tokenizer, is_chat_model, model_type)
+
+if __name__ == '__main__':
+  fire.Fire(main)

llama_chat_format.py

@@ -0,0 +1,38 @@
+BOS, EOS = "<s>", "</s>"
+B_INST, E_INST = "[INST]", "[/INST]"
+B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"
+DEFAULT_SYSTEM_PROMPT = """\
+You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."""
+
+def format_to_llama_chat_style(history) -> str:
+    # history has the following structure:
+    # - dialogs
+    # --- instruction
+    # --- response (None for the most recent dialog)
+    prompt = ""
+    for i, dialog in enumerate(history[:-1]):
+      instruction, response = dialog[0], dialog[1]
+      # prepend system instruction before first instruction
+      if i == 0:
+        instruction = f"{B_SYS}{DEFAULT_SYSTEM_PROMPT}{E_SYS}" + instruction
+      else:
+        # the tokenizer automatically adds a bos_token during encoding,
+        # for this reason the bos_token is not added for the first instruction
+        prompt += BOS
+      prompt += f"{B_INST} {instruction.strip()} {E_INST} {response.strip()} " + EOS
+
+    # new instruction from the user
+    new_instruction = history[-1][0].strip()
+
+    # the tokenizer automatically adds a bos_token during encoding,
+    # for this reason the bos_token is not added for the first instruction
+    if len(history) > 1:
+      prompt += BOS
+    else:
+      # prepend system instruction before first instruction
+      new_instruction = f"{B_SYS}{DEFAULT_SYSTEM_PROMPT}{E_SYS}" + new_instruction
+
+    prompt += f"{B_INST} {new_instruction} {E_INST}"
+    return prompt

requirements.txt

@@ -0,0 +1,6 @@
+transformers==4.31.0
+auto-gptq==0.3.0
+langchain==0.0.237
+gradio==3.37.0
+llama-cpp-python==0.1.73
+fire==0.5.0