Upload folder using huggingface_hub

config.json

@@ -1,4 +1,5 @@
 {
+  "_name_or_path": "360Zhinao-7B-Chat-360K",
   "architectures": [
     "ZhinaoForCausalLM"
   ],
@@ -6,8 +7,8 @@
     "AutoConfig": "configuration_zhinao.ZhinaoConfig",
     "AutoModelForCausalLM": "modeling_zhinao.ZhinaoForCausalLM"
   },
-  "bf16": true,
-  "fp16": false,
+  "bf16": false,
+  "fp16": true,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.01,
@@ -19,25 +20,35 @@
   "num_hidden_layers": 32,
   "num_key_value_heads": 32,
   "quantization_config": {
+    "batch_size": 1,
     "bits": 4,
-    "checkpoint_format": "gptq",
-    "damp_percent": 0.01,
+    "block_name_to_quantize": null,
+    "cache_block_outputs": true,
+    "damp_percent": 0.1,
     "desc_act": false,
+    "exllama_config": {
+      "version": 1
+    },
     "group_size": 128,
-    "model_file_base_name": "gptq_model-4bit-128g",
-    "model_name_or_path": "360Zhinao-7B-Chat-360K-Int4",
+    "max_input_length": null,
+    "model_seqlen": null,
+    "module_name_preceding_first_block": null,
+    "modules_in_block_to_quantize": null,
+    "pad_token_id": null,
     "quant_method": "gptq",
-    "static_groups": false,
     "sym": true,
-    "true_sequential": true
+    "tokenizer": null,
+    "true_sequential": true,
+    "use_cuda_fp16": false,
+    "use_exllama": true
   },
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "rope_theta": 50000000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "float16",
-  "transformers_version": "4.38.2",
+  "transformers_version": "4.38.1",
   "use_cache": false,
-  "use_flash_attn": "auto",
+  "use_flash_attn": true,
   "vocab_size": 158464
 }

generation_config.json

@@ -1,12 +1,14 @@
 {
-    "do_sample": true,
-    "top_k": 0,
-    "top_p": 0.8,
-    "temperature": 1.0,
-    "repetition_penalty": 1.0,
-    "pad_token_id": 158326,
-    "eos_token_id": [158326, 158332, 158333],
-    "max_new_tokens": 1024,
-    "_from_model_config": true,
-    "transformers_version": "4.38.2"
-}
+  "_from_model_config": true,
+  "do_sample": true,
+  "eos_token_id": [
+    158326,
+    158332,
+    158333
+  ],
+  "max_new_tokens": 1024,
+  "pad_token_id": 158326,
+  "top_k": 0,
+  "top_p": 0.8,
+  "transformers_version": "4.38.1"
+}

generation_utils.py

@@ -1,3 +1,4 @@
+import re
 import torch
 import numpy as np
 from queue import Queue
@@ -76,14 +77,75 @@ def make_context(model, tokenizer,
     return torch.LongTensor([input_tokens]).to(model.device)
 
 
+def parse_pot_no_stream(inputs):
+    try:
+        s = re.findall(r'<<(.*?)>>', inputs, re.DOTALL)
+        if not s:
+            #print("err inputs: ", origin_inputs, flush=True)
+            return inputs
+
+        index = 0
+        for k in s:
+            try:
+                if "func" in k:
+                    var = k.split("=", 1)
+                    try:
+                        var[1] = var[1].strip(" ")
+                        exec(var[1], globals())
+                        ans = func()
+                    except:
+                        if 'sympy' in var[1]:
+                            var[1] = var[1].replace('res[x]', 'res[0][0]').replace('res[y]', 'res[0][1]')
+                            exec(var[1], globals())
+                            ans = func()
+                        pass
+                    var_list = [c.strip(" ") for c in var[0].split(",")]
+                    if len(var_list) == 1:
+                        ans = [ans]
+
+                    for i in range(len(ans)):
+                        try:
+                            ans[i] = float(ans[i])
+                            if abs(ans[i] - int(ans[i])) < 1e-10:
+                                ans[i] = str(int(ans[i]))
+                        except:
+                            pass
+
+                    inputs = inputs.replace("<<"+k+">>", "")
+                    for i in range(len(var_list)):
+                        inputs = inputs.replace(var_list[i], str(ans[i]))
+                    index += 1
+                    for c in range(index, len(s)):
+                        for i in range(len(var_list)):
+                            s[c] = s[c].replace(var_list[i], str(ans[i]))
+                else:
+                    var = k.replace(" ", "").split("=")
+                    var[1] = var[1].replace("eval", "")
+                    ans = round(eval(var[1]), 10)
+                    ans = float(ans)
+                    if abs(ans - int(ans)) < 1e-10:
+                        ans = str(int(ans))
+                    inputs = inputs.replace("<<"+k+">>", "").replace(var[0], str(ans))
+                    index += 1
+                    for c in range(index, len(s)):
+                        s[c] = s[c].replace(var[0], str(ans))
+            except:
+                return inputs
+    except Exception as e:
+        return inputs 
+
+    return inputs
+
+
 class TextIterStreamer:
-    def __init__(self, tokenizer, skip_prompt=False, skip_special_tokens=False):
+    def __init__(self, tokenizer, skip_prompt=False, skip_special_tokens=False, use_pot=True):
         self.tokenizer = tokenizer
         self.skip_prompt = skip_prompt
         self.skip_special_tokens = skip_special_tokens
         self.tokens = []
         self.text_queue = Queue()
         self.next_tokens_are_prompt = True
+        self.use_pot = use_pot
 
     def put(self, value):
         if self.skip_prompt and self.next_tokens_are_prompt:
@@ -92,8 +154,10 @@ class TextIterStreamer:
             if len(value.shape) > 1:
                 value = value[0]
             self.tokens.extend(value.tolist())
-            self.text_queue.put(
-                self.tokenizer.decode(self.tokens, skip_special_tokens=self.skip_special_tokens, errors='ignore'))
+            tokens_str = self.tokenizer.decode(self.tokens, skip_special_tokens=self.skip_special_tokens, errors='ignore')
+            if self.use_pot:
+                tokens_str = parse_pot_no_stream(tokens_str)
+            self.text_queue.put(tokens_str)
 
     def end(self):
         self.text_queue.put(None)

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbe0fa0270558e006f3044a51a93fddfde2a63e295b39cd6cf12bd8ef8de631f
+size 1988671752

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f4ad0f37305494c798be942f64caa426436576a40b0d26595a281fee8978d75
+size 1978251896

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de6a6e3aa38f51a610fc77dc697f389172caebadf23692972751e61fb47bb117
+size 702555592

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aa9f6913ddd449fc02dafa29f2dbc548ac52f3f3b50b592be2a1e067944c694
+size 1298137216

model.safetensors.index.json

@@ -0,0 +1,874 @@
+{
+  "metadata": {
+    "total_size": 5967519744
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.qkv_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.qkv_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.qkv_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.qkv_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.qkv_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.qkv_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.qkv_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.qkv_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.qkv_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.qkv_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.qkv_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.qkv_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.qkv_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.qkv_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.qkv_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.23.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.24.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.25.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.25.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.25.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.25.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.25.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.qkv_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.qkv_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.qkv_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.qkv_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.qkv_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.qkv_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.qkv_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.qkv_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.qkv_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.qkv_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.qkv_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.qkv_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.qkv_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.qkv_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.qkv_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.qkv_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.qkv_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.qkv_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.qkv_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.qkv_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.qkv_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.qkv_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.qkv_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.qkv_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.qkv_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.qkv_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.qkv_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.qkv_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.qkv_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.qkv_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.qkv_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.qkv_proj.g_idx": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.qkv_proj.qweight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.qkv_proj.qzeros": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.qkv_proj.scales": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.qkv_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.qkv_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.qkv_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.qkv_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.qkv_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.qkv_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.qkv_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.qkv_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.qkv_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.qkv_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.qkv_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.qkv_proj.g_idx": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.qkv_proj.qweight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.qkv_proj.qzeros": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.qkv_proj.scales": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.scales": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.qkv_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.qkv_proj.g_idx": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.qkv_proj.qweight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.qkv_proj.qzeros": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.qkv_proj.scales": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

modeling_zhinao.py

@@ -19,7 +19,7 @@ from transformers.utils import logging
 from transformers.generation.utils import GenerationConfig
 from transformers.generation.logits_process import LogitsProcessorList
 from .configuration_zhinao import ZhinaoConfig
-from .generation_utils import TextIterStreamer, make_context, OutputRepetitionPenaltyLogitsProcessor
+from .generation_utils import TextIterStreamer, make_context, OutputRepetitionPenaltyLogitsProcessor, parse_pot_no_stream
 
  
 try:
@@ -932,6 +932,7 @@ class ZhinaoForCausalLM(ZhinaoPreTrainedModel):
             messages: List[dict],
             system: str = "You are a helpful assistant.",
             stream=False,
+            use_pot=True,
             generation_config: Optional[GenerationConfig]=None):
         
         generation_config = generation_config or self.generation_config
@@ -941,7 +942,7 @@ class ZhinaoForCausalLM(ZhinaoPreTrainedModel):
         )
 
         if stream:
-            streamer = TextIterStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+            streamer = TextIterStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True, use_pot=use_pot)
             Thread(target=self.generate, kwargs=dict(
                 inputs=input_ids, streamer=streamer,
                 generation_config=generation_config,
@@ -950,8 +951,9 @@ class ZhinaoForCausalLM(ZhinaoPreTrainedModel):
         else:
             outputs = self.generate(input_ids, generation_config=generation_config)
             response = tokenizer.decode(outputs[0][len(input_ids[0]):], skip_special_tokens=True)
-        
-        return response
+            if use_pot:
+                response = parse_pot_no_stream(response)
+            return response
 
 
 class ZhinaoForSequenceClassification(ZhinaoPreTrainedModel):

quantize_config.json

@@ -1,13 +1,23 @@
 {
+  "batch_size": 1,
   "bits": 4,
-  "group_size": 128,
-  "damp_percent": 0.01,
+  "block_name_to_quantize": null,
+  "cache_block_outputs": true,
+  "damp_percent": 0.1,
   "desc_act": false,
-  "static_groups": false,
+  "exllama_config": {
+    "version": 1
+  },
+  "group_size": 128,
+  "max_input_length": null,
+  "model_seqlen": null,
+  "module_name_preceding_first_block": null,
+  "modules_in_block_to_quantize": null,
+  "pad_token_id": null,
+  "quant_method": "gptq",
   "sym": true,
+  "tokenizer": null,
   "true_sequential": true,
-  "model_name_or_path": "360Zhinao-7B-Chat-360K-Int4",
-  "model_file_base_name": "gptq_model-4bit-128g",
-  "quant_method": "gptq",
-  "checkpoint_format": "gptq"
+  "use_cuda_fp16": false,
+  "use_exllama": true
 }

special_tokens_map.json

@@ -0,0 +1,3 @@
+{
+  "pad_token": "<pad>"
+}

tokenizer_config.json

@@ -6,6 +6,16 @@
       null
     ]
   },
+  "stop_token_ids": [
+        158326,
+        158332,
+        158333
+  ],
+  "stop": [
+        "<eod>",
+        "<|im_start|>",
+        "<|im_end|>"
+  ],
   "clean_up_tokenization_spaces": false,
   "do_lower_case": false,
   "eos_token": "<eod>",
@@ -15,4 +25,4 @@
   "remove_space": false,
   "tokenizer_class": "ZhinaoTokenizer",
   "unk_token": "<unk>"
-}
+}