willhe-xverse commited on
Commit
e2bfb2b
1 Parent(s): c444ad6

Upload folder using huggingface_hub

Browse files
config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "XverseForCausalLM"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_xverse.XverseConfig",
7
+ "AutoModelForCausalLM": "modeling_xverse.XverseForCausalLM"
8
+ },
9
+ "bos_token_id": 2,
10
+ "eos_token_id": 3,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 8192,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 22016,
15
+ "max_position_embeddings": 8192,
16
+ "max_tokenizer_truncation": 6144,
17
+ "model_type": "xverse",
18
+ "num_attention_heads": 64,
19
+ "num_hidden_layers": 80,
20
+ "pad_token_id": 1,
21
+ "quantization_config": {
22
+ "bits": 4,
23
+ "damp_percent": 0.01,
24
+ "desc_act": true,
25
+ "group_size": 128,
26
+ "is_marlin_format": false,
27
+ "model_file_base_name": "gptq_model-4bit-128g",
28
+ "model_name_or_path": "/mnt/llm_dataset/xverse_models/gptq/XVERSE-65B-Chat-GPTQ-Int4",
29
+ "quant_method": "gptq",
30
+ "static_groups": true,
31
+ "sym": true,
32
+ "true_sequential": true
33
+ },
34
+ "rms_norm_eps": 1e-06,
35
+ "tie_word_embeddings": false,
36
+ "torch_dtype": "float16",
37
+ "transformers_version": "4.38.2",
38
+ "use_cache": true,
39
+ "vocab_size": 100534
40
+ }
configuration_xverse.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+ # and OPT implementations in this library. It has been modified from its
6
+ # original forms to accommodate minor architectural differences compared
7
+ # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+ #
9
+ # Licensed under the Apache License, Version 2.0 (the "License");
10
+ # you may not use this file except in compliance with the License.
11
+ # You may obtain a copy of the License at
12
+ #
13
+ # http://www.apache.org/licenses/LICENSE-2.0
14
+ #
15
+ # Unless required by applicable law or agreed to in writing, software
16
+ # distributed under the License is distributed on an "AS IS" BASIS,
17
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ # See the License for the specific language governing permissions and
19
+ # limitations under the License.
20
+ """ XVERSE model configuration"""
21
+
22
+ from transformers.configuration_utils import PretrainedConfig
23
+ from transformers.utils import logging
24
+
25
+
26
+ logger = logging.get_logger(__name__)
27
+
28
+ XVERSE_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
29
+
30
+
31
+ class XverseConfig(PretrainedConfig):
32
+ r"""
33
+ This is the configuration class to store the configuration of a [`XverseModel`]. It is used to instantiate an Xverse
34
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
35
+ defaults will yield a similar configuration to that of the XVERSE-13B.
36
+
37
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
38
+ documentation from [`PretrainedConfig`] for more information.
39
+
40
+
41
+ Args:
42
+ vocab_size (`int`, *optional*, defaults to 100278):
43
+ Vocabulary size of the XVERSE model. Defines the number of different tokens that can be represented by the
44
+ `inputs_ids` passed when calling [`XverseModel`]
45
+ hidden_size (`int`, *optional*, defaults to 5120):
46
+ Dimension of the hidden representations.
47
+ intermediate_size (`int`, *optional*, defaults to 13824):
48
+ Dimension of the MLP representations.
49
+ num_hidden_layers (`int`, *optional*, defaults to 40):
50
+ Number of hidden layers in the Transformer encoder.
51
+ num_attention_heads (`int`, *optional*, defaults to 40):
52
+ Number of attention heads for each attention layer in the Transformer encoder.
53
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
54
+ The non-linear activation function (function or string) in the decoder.
55
+ max_position_embeddings (`int`, *optional*, defaults to 8192):
56
+ The maximum sequence length that this model might ever be used with. Typically set this to something large
57
+ just in case (e.g., 512 or 1024 or 2048).
58
+ initializer_range (`float`, *optional*, defaults to 0.02):
59
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
60
+ rms_norm_eps (`float`, *optional*, defaults to 1e-6):
61
+ The epsilon used by the rms normalization layers.
62
+ use_cache (`bool`, *optional*, defaults to `True`):
63
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
64
+ relevant if `config.is_decoder=True`.
65
+ tie_word_embeddings(`bool`, *optional*, defaults to `False`):
66
+ Whether to tie weight embeddings
67
+
68
+ Example:
69
+
70
+ ```python
71
+ >>> from transformers import XverseModel, XverseConfig
72
+
73
+ >>> # Initializing a Xverse XVERSE-13B style configuration
74
+ >>> configuration = XverseConfig()
75
+
76
+ >>> # Initializing a model from the XVERSE-13B style configuration
77
+ >>> model = XverseModel(configuration)
78
+
79
+ >>> # Accessing the model configuration
80
+ >>> configuration = model.config
81
+ ```"""
82
+ model_type = "xverse"
83
+ keys_to_ignore_at_inference = ["past_key_values"]
84
+
85
+ def __init__(
86
+ self,
87
+ vocab_size=100278,
88
+ hidden_size=5120,
89
+ intermediate_size=13824,
90
+ num_hidden_layers=40,
91
+ num_attention_heads=40,
92
+ hidden_act="silu",
93
+ max_position_embeddings=8192,
94
+ max_tokenizer_truncation=8192,
95
+ initializer_range=0.02,
96
+ rms_norm_eps=1e-6,
97
+ use_cache=True,
98
+ pad_token_id=None,
99
+ bos_token_id=1,
100
+ eos_token_id=2,
101
+ tie_word_embeddings=False,
102
+ **kwargs,
103
+ ):
104
+ self.vocab_size = vocab_size
105
+ self.max_position_embeddings = max_position_embeddings
106
+ self.hidden_size = hidden_size
107
+ self.intermediate_size = intermediate_size
108
+ self.num_hidden_layers = num_hidden_layers
109
+ self.num_attention_heads = num_attention_heads
110
+
111
+ self.hidden_act = hidden_act
112
+ self.initializer_range = initializer_range
113
+ self.rms_norm_eps = rms_norm_eps
114
+ self.use_cache = use_cache
115
+ self.max_tokenizer_truncation = max_tokenizer_truncation
116
+
117
+ super().__init__(
118
+ pad_token_id=pad_token_id,
119
+ bos_token_id=bos_token_id,
120
+ eos_token_id=eos_token_id,
121
+ tie_word_embeddings=tie_word_embeddings,
122
+ **kwargs,
123
+ )
gptq_model-4bit-128g.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d5a74c7f357e2946debd6186d7ff4a67b36b20d498539775fabf58ed7d8b6185
3
+ size 36978547264
quantize_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bits": 4,
3
+ "group_size": 128,
4
+ "damp_percent": 0.01,
5
+ "desc_act": true,
6
+ "static_groups": true,
7
+ "sym": true,
8
+ "true_sequential": true,
9
+ "model_name_or_path": "/mnt/llm_dataset/xverse_models/gptq/XVERSE-65B-Chat-GPTQ-Int4/",
10
+ "model_file_base_name": "gptq_model-4bit-128g",
11
+ "is_marlin_format": false,
12
+ "quant_method": "gptq"
13
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|startoftext|>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<pad>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "clean_up_tokenization_spaces": true,
3
+ "model_max_length": 1000000000000000019884624838656,
4
+ "tokenizer_class": "PreTrainedTokenizerFast"
5
+ }