model upload

config.json

@@ -0,0 +1,44 @@
+{
+  "_name_or_path": "/data04/ysd4/llama/Qwen-72B",
+  "architectures": [
+    "QWenLMHeadModel"
+  ],
+  "attn_dropout_prob": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_qwen.QWenConfig",
+    "AutoModel": "modeling_qwen.QWenLMHeadModel",
+    "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel"
+  },
+  "bf16": false,
+  "emb_dropout_prob": 0.0,
+  "fp16": true,
+  "fp32": false,
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 49152,
+  "kv_channels": 128,
+  "layer_norm_epsilon": 1e-06,
+  "max_position_embeddings": 32768,
+  "model_type": "qwen",
+  "no_bias": true,
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "onnx_safe": null,
+  "rope_theta": 1000000,
+  "rotary_emb_base": 1000000,
+  "rotary_pct": 1.0,
+  "scale_attn_weights": true,
+  "seq_length": 32768,
+  "softmax_in_fp32": false,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "QWenTokenizer",
+  "torch_dtype": "float16",
+  "transformers_version": "4.34.1",
+  "use_cache": true,
+  "use_cache_kernel": false,
+  "use_cache_quantization": false,
+  "use_dynamic_ntk": false,
+  "use_flash_attn": true,
+  "use_logn_attn": false,
+  "vocab_size": 152064
+}

configuration_qwen.py

@@ -0,0 +1,71 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from transformers import PretrainedConfig
+
+
+class QWenConfig(PretrainedConfig):
+    model_type = "qwen"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=151936,
+        hidden_size=4096,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        emb_dropout_prob=0.0,
+        attn_dropout_prob=0.0,
+        layer_norm_epsilon=1e-6,
+        initializer_range=0.02,
+        max_position_embeddings=8192,
+        scale_attn_weights=True,
+        use_cache=True,
+        bf16=False,
+        fp16=False,
+        fp32=False,
+        kv_channels=128,
+        rotary_pct=1.0,
+        rotary_emb_base=10000,
+        use_dynamic_ntk=True,
+        use_logn_attn=True,
+        use_flash_attn="auto",
+        intermediate_size=22016,
+        no_bias=True,
+        tie_word_embeddings=False,
+        use_cache_quantization=False,
+        use_cache_kernel=False,
+        softmax_in_fp32=False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.emb_dropout_prob = emb_dropout_prob
+        self.attn_dropout_prob = attn_dropout_prob
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_range = initializer_range
+        self.scale_attn_weights = scale_attn_weights
+        self.use_cache = use_cache
+        self.max_position_embeddings = max_position_embeddings
+        self.bf16 = bf16
+        self.fp16 = fp16
+        self.fp32 = fp32
+        self.kv_channels = kv_channels
+        self.rotary_pct = rotary_pct
+        self.rotary_emb_base = rotary_emb_base
+        self.use_dynamic_ntk = use_dynamic_ntk
+        self.use_logn_attn = use_logn_attn
+        self.use_flash_attn = use_flash_attn
+        self.no_bias = no_bias
+        self.use_cache_quantization = use_cache_quantization
+        self.use_cache_kernel = use_cache_kernel
+        self.softmax_in_fp32 = softmax_in_fp32
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs
+        )

cpp_kernels.py

@@ -0,0 +1,55 @@
+from torch.utils import cpp_extension
+import pathlib
+import os
+import subprocess
+
+def _get_cuda_bare_metal_version(cuda_dir):
+    raw_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
+                                         universal_newlines=True)
+    output = raw_output.split()
+    release_idx = output.index("release") + 1
+    release = output[release_idx].split(".")
+    bare_metal_major = release[0]
+    bare_metal_minor = release[1][0]
+
+    return raw_output, bare_metal_major, bare_metal_minor
+
+def _create_build_dir(buildpath):
+    try:
+        os.mkdir(buildpath)
+    except OSError:
+        if not os.path.isdir(buildpath):
+            print(f"Creation of the build directory {buildpath} failed")
+
+# Check if cuda 11 is installed for compute capability 8.0
+cc_flag = []
+_, bare_metal_major, bare_metal_minor = _get_cuda_bare_metal_version(cpp_extension.CUDA_HOME)
+if int(bare_metal_major) >= 11:
+    cc_flag.append('-gencode')
+    cc_flag.append('arch=compute_80,code=sm_80')
+    if int(bare_metal_minor) >= 7:
+        cc_flag.append('-gencode')
+        cc_flag.append('arch=compute_90,code=sm_90')
+
+# Build path
+srcpath = pathlib.Path(__file__).parent.absolute()
+buildpath = srcpath / 'build'
+_create_build_dir(buildpath)
+
+def _cpp_extention_load_helper(name, sources, extra_cuda_flags):
+    return cpp_extension.load(
+        name=name,
+        sources=sources,
+        build_directory=buildpath,
+        extra_cflags=['-O3', ],
+        extra_cuda_cflags=['-O3',
+                           '-gencode', 'arch=compute_70,code=sm_70',
+                           '--use_fast_math'] + extra_cuda_flags + cc_flag,
+        verbose=1
+    )
+
+extra_flags = []
+
+cache_autogptq_cuda_256_sources = ["./cache_autogptq_cuda_256.cpp",
+           "./cache_autogptq_cuda_kernel_256.cu"]
+cache_autogptq_cuda_256 = _cpp_extention_load_helper("cache_autogptq_cuda_256", cache_autogptq_cuda_256_sources, extra_flags)

generation_config.json

@@ -0,0 +1,15 @@
+{
+  "chat_format": "raw",
+  "do_sample": true,
+  "eos_token_id": 151643,
+  "max_new_tokens": 512,
+  "pad_token_id": 151643,
+  "stop_words_ids": [
+    [
+      151643
+    ]
+  ],
+  "top_k": 0,
+  "top_p": 0.8,
+  "transformers_version": "4.34.1"
+}

model-00001-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87c2c0cbc5c661fbf926a59dc4f61c22e8241b8fa610b13e5a6338efa8add2cd
+size 2491416712

model-00002-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe141c7efd46bfb1e54f7819771a1fbada6c4feab1c154564c292e930bb1983f
+size 939606672

model-00003-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b00be35ad7547ba722a6dfe148df3abeda6c7e39e8ce55dbb12e0335c92cd51f
+size 805323104

model-00004-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29eecfa6621b7d6b8509e72fd59929ba0d28b91cd8682902d36ce3ad9da7153d
+size 939590192

model-00005-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:383dd555f9cf515c774f79fdae93c95a8cea0b194e277572f55baaae9db36fff
+size 805323104

model-00006-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1046a9861391c9ea878db11bf54507eb197b50cd9b6338f1e0dfce6be6cf6ef0
+size 939590192

model-00007-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:301d90da63af5a6aac8f0d4055d0d79281ed8b887918aa0a3e4b8dc6c799ad28
+size 805323104

model-00008-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:004ac816039458ea54753219cd3301ee8e5be8d3f4d401912bcad29d42d8c6f2
+size 939590192

model-00009-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ebe03d8e7e6a42fbdcb714f20baba67c8a85d46d4e4b9405c50a653e887454d
+size 805323104

model-00010-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b26df13b1119abb204c69515929e955f748e24fcfc2fde3d3f72e9f5273ae73
+size 939590192

model-00011-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a1312c0d69d65a6bcaaa19a3e1143260f3927b2ace4baf0ed0dc20ce563a7a4
+size 805323104

model-00012-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ca873a5fc42bc29d62843869a17e40b4435f82cbb63eb7c4b5e79a8f86e3701
+size 939590192

model-00013-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf2ee53bf11b9a7a820a4440bdcc97dbfeb82dc7b8c4cbe1fb856d032907f687
+size 805323104

model-00014-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb1e4b4b235af2131a1f272d0fa1fbdd6b4b6069d0e01881405cea6f3b45dd81
+size 939590192

model-00015-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03a076df233722df0884a9cfc0a892d65c1f7ce8d2be515449897bb587f2f689
+size 805323104

model-00016-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b067e38235385662bf563f7e545a632ab1bbd849ab6b020d293888f13b09bbcc
+size 939590192

model-00017-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca09d7cddf9b2aec0e1c122182b9bf595299c54f8b0d9457574e486e3c199bb9
+size 805323104

model-00018-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a435728bbe8b3314c48e7178e753c13e4a0d4db2b5b0c7e76d52015352af27a
+size 939590192

model-00019-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d1f322d92ee837645ea2fbe121ef7272f8b68f6cf51c7b35251507959c34aad
+size 805323104

model-00020-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63fa8f6ea9f88d2f390ed01d2d93432b91c69cd058b15fa46de2a079a0c668b8
+size 939590192

model-00021-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc615249b8c6df9ec0c5b82069401deb1487fb782869fdae91ae5699df826eb0
+size 805323096

model-00022-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08a4de09bb9d125ad6f3c67acb6926340328082627418d9afcc6830f23a4361c
+size 939590192

model-00023-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a62dbe2203990155c7ec3e9b4a87105dc0c8c7addddd299f6485e48bc5bbc733
+size 805323104

model-00024-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:747584b0cc78f98e8ff398518846de5c4404786d0ec4b9b47a7509bf068e9596
+size 939590192

model-00025-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d466e745c2a64206602e4a3900275e2bf167db4fcea1d7c43bdfe388ece400b
+size 805323104

model-00026-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:858f7a96c503539fb1262164d10106d3cb5419177f05c23c3c061f3614391490
+size 939590192

model-00027-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70fd1cce68239ce06a5e71497e460f0567df62be2a943a11b3996445a04bd68a
+size 805323104

model-00028-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c86d3e1d2f7ff82f32a23c666cf90b74b77c943aee759a4d3b5792fc1c0cb22d
+size 939590192

model-00029-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b5bc2a4e707521b4902c561e43f05cfdf32cb4bf80b4f4a0ff2ebbf51063964
+size 805323104

model-00030-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5d76c01d27fb6c77b3dfe368a6a3408e95c4cb5abb3ebc0481a2d02214c08f2
+size 939590192

model-00031-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8c7cd12f8faefc2e7177e9e039edc0eb0e1bf463c64e9536fc8a57b3aa1e655
+size 805323104

model-00032-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dcb13b7b355cff9100aad1b86e322aa65582d18f086b66d05e0400e5709f5d7
+size 939590192

model-00033-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:605670bc7373c9d73e83916c32cdb024a7406082cad20b2c0a153f722a56fe76
+size 805323104

model-00034-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a920b96946b2e984941c17db460a14c560718ba6124bb61c4eda31456df98d1
+size 939590192

model-00035-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf4ff7ac3e1ffbbe4cf9bd607ca5731f35ec42a909930a42d85f91aaa14be7bb
+size 805323104

model-00036-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c83da519cee9de7d14ff1ebacc885e735e0ab9e7e588105cb4d9db3554cdc31c
+size 939590192

model-00037-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af786946b2cd84b084d545bfc14e8000d5fee9f29af9d3589dbb94420c180901
+size 805323104

model-00038-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32b7a24693ca1457d48f696e11c35caecfc335ba66e45e736633a22bcac12dc2
+size 939590192

model-00039-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8b3352588fb17fed9fff827f3b5812393ac79a80f1e320c1953c11c93aaebf6
+size 805323104

model-00040-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1df49b1c471c9e0e39564b59b9d9188e5f97e46961cb7eff3f2c00dfbb3de96
+size 939590192

model-00041-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca2483206e26d077cdb44b3dbfb6efd2bf2c5ed27d6a258e2f0a45080279330b
+size 805323104

model-00042-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8441e90341c1229cc7d835ce364af103fd830a21ef1416db4a791f6c1734c228
+size 939590192

model-00043-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96b0f22f953c5c4b9c3b10ebedc1298dac62a37e6afdb953ea92c8e1f17fec83
+size 805323104

model-00044-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17f18203b36d2027e837b126308d8adb7ea9c489c8e5c1f8a748c0cf4a2ad47c
+size 939590192

model-00045-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:262c0f911495a9b4451a10731a6cd4b3978d629e2022e6ee12a68a7b71dfde39
+size 805323104

model-00046-of-00162.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6da1d99db87a56b3cf225af665a8dd83b011dd4dab236468d416e82ae6e46c5c
+size 939590192