Xidong commited on
Commit
193ca41
1 Parent(s): 3104057

Delete Apollo-34B

Browse files
Apollo-34B/config.json DELETED
@@ -1,26 +0,0 @@
1
- {
2
- "architectures": [
3
- "LlamaForCausalLM"
4
- ],
5
- "bos_token_id": 1,
6
- "eos_token_id": 2,
7
- "hidden_act": "silu",
8
- "hidden_size": 7168,
9
- "initializer_range": 0.02,
10
- "intermediate_size": 20480,
11
- "max_position_embeddings": 4096,
12
- "model_type": "llama",
13
- "num_attention_heads": 56,
14
- "num_hidden_layers": 60,
15
- "num_key_value_heads": 8,
16
- "pad_token_id": 0,
17
- "pretraining_tp": 1,
18
- "rms_norm_eps": 1e-05,
19
- "rope_scaling": null,
20
- "rope_theta": 5000000.0,
21
- "tie_word_embeddings": false,
22
- "torch_dtype": "bfloat16",
23
- "transformers_version": "4.34.0",
24
- "use_cache": true,
25
- "vocab_size": 64000
26
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
Apollo-34B/configuration_yi.py DELETED
@@ -1,121 +0,0 @@
1
- """ Yi model configuration"""
2
- from transformers.configuration_utils import PretrainedConfig
3
- from transformers.utils import logging
4
-
5
- logger = logging.get_logger(__name__)
6
-
7
- Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8
-
9
-
10
- class YiConfig(PretrainedConfig):
11
- r"""
12
- This is the configuration class to store the configuration of a [`YiModel`]. It is used to instantiate an Yi
13
- model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
14
- defaults will yield a similar configuration to that of the Yi model.
15
-
16
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
17
- documentation from [`PretrainedConfig`] for more information.
18
-
19
-
20
- Args:
21
- vocab_size (`int`, *optional*, defaults to 64000):
22
- Vocabulary size of the Yi model. Defines the number of different tokens that can be represented by the
23
- `inputs_ids` passed when calling [`YiModel`]
24
- hidden_size (`int`, *optional*, defaults to 4096):
25
- Dimension of the hidden representations.
26
- intermediate_size (`int`, *optional*, defaults to 11008):
27
- Dimension of the MLP representations.
28
- num_hidden_layers (`int`, *optional*, defaults to 32):
29
- Number of hidden layers in the Transformer encoder.
30
- num_attention_heads (`int`, *optional*, defaults to 32):
31
- Number of attention heads for each attention layer in the Transformer encoder.
32
- num_key_value_heads (`int`, *optional*):
33
- This is the number of key_value heads that should be used to implement Grouped Query Attention. If
34
- `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
35
- `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
36
- converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
37
- by meanpooling all the original heads within that group. For more details checkout [this
38
- paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
39
- `num_attention_heads`.
40
- hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
41
- The non-linear activation function (function or string) in the decoder.
42
- max_position_embeddings (`int`, *optional*, defaults to 4096):
43
- The maximum sequence length that this model might ever be used with. Typically set this to something large
44
- just in case (e.g., 512 or 1024 or 2048 or 4096).
45
- initializer_range (`float`, *optional*, defaults to 0.02):
46
- The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
47
- rms_norm_eps (`float`, *optional*, defaults to 1e-5):
48
- The epsilon used by the rms normalization layers.
49
- use_cache (`bool`, *optional*, defaults to `True`):
50
- Whether or not the model should return the last key/values attentions (not used by all models). Only
51
- relevant if `config.is_decoder=True`.
52
- tie_word_embeddings(`bool`, *optional*, defaults to `False`):
53
- Whether to tie weight embeddings
54
- output_attentions (`bool`, *optional*, defaults to `False`):
55
- Whether or not to output attentions.
56
- rope_theta (`float`, *optional*, defaults to 5000000.0):
57
- The base period of the RoPE embeddings.
58
- Example:
59
-
60
- ```python
61
- >>> from transformers import YiModel, YiConfig
62
-
63
- >>> # Initializing a Yi style configuration
64
- >>> configuration = YiConfig()
65
-
66
- >>> # Initializing a model from the Yi style configuration
67
- >>> model = YiModel(configuration)
68
-
69
- >>> # Accessing the model configuration
70
- >>> configuration = model.config
71
- ```"""
72
- model_type = "Yi"
73
- keys_to_ignore_at_inference = ["past_key_values"]
74
-
75
- def __init__(
76
- self,
77
- vocab_size=64000,
78
- hidden_size=4096,
79
- intermediate_size=11008,
80
- num_hidden_layers=32,
81
- num_attention_heads=32,
82
- num_key_value_heads=4,
83
- hidden_act="silu",
84
- max_position_embeddings=4096,
85
- initializer_range=0.02,
86
- rms_norm_eps=1e-5,
87
- use_cache=True,
88
- pad_token_id=0,
89
- bos_token_id=1,
90
- eos_token_id=2,
91
- tie_word_embeddings=False,
92
- output_attentions=False,
93
- rope_theta=5000000.0,
94
- **kwargs,
95
- ):
96
- self.vocab_size = vocab_size
97
- self.max_position_embeddings = max_position_embeddings
98
- self.hidden_size = hidden_size
99
- self.intermediate_size = intermediate_size
100
- self.num_hidden_layers = num_hidden_layers
101
- self.num_attention_heads = num_attention_heads
102
-
103
- # for backward compatibility
104
- if num_key_value_heads is None:
105
- num_key_value_heads = num_attention_heads
106
-
107
- self.num_key_value_heads = num_key_value_heads
108
- self.hidden_act = hidden_act
109
- self.initializer_range = initializer_range
110
- self.rms_norm_eps = rms_norm_eps
111
- self.use_cache = use_cache
112
- self.output_attentions = output_attentions
113
- self.rope_theta = rope_theta
114
-
115
- super().__init__(
116
- pad_token_id=pad_token_id,
117
- bos_token_id=bos_token_id,
118
- eos_token_id=eos_token_id,
119
- tie_word_embeddings=tie_word_embeddings,
120
- **kwargs,
121
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
Apollo-34B/generation_config.json DELETED
@@ -1,7 +0,0 @@
1
- {
2
- "_from_model_config": true,
3
- "bos_token_id": 1,
4
- "eos_token_id": 2,
5
- "pad_token_id": 0,
6
- "transformers_version": "4.35.2"
7
- }
 
 
 
 
 
 
 
 
Apollo-34B/model-00001-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:68c97fec044d5911d17a166e050f5022c18d87651e4fe69d9b5a906ecf79794d
3
- size 4793130656
 
 
 
 
Apollo-34B/model-00002-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:905a63b0bd8be9865299244a8ce485251606bed1e1cd0bbe0813865cdee9d64e
3
- size 4756459576
 
 
 
 
Apollo-34B/model-00003-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d953fefe2590116885751effecf7fdd4aca4965691a8e0dbe3639ccd26f6a357
3
- size 4991369960
 
 
 
 
Apollo-34B/model-00004-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:306c36bdea6ec73124f1573a02866b3f24474acd4d4f5e7d25b9e20a63acb8a6
3
- size 4756459616
 
 
 
 
Apollo-34B/model-00005-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:21fa00391eef98351b5d3caad834d878cf89c6a463cdc3c7c0af2a3f01bfd2f6
3
- size 4756459616
 
 
 
 
Apollo-34B/model-00006-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e0c40ca1ad995c0e39b4727e4b38dad6f3db15587e1661cf30fcf0a048a5211b
3
- size 4991369984
 
 
 
 
Apollo-34B/model-00007-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:61ee60da8e93e89af966faef94484672fc78aac166df1702f27da4518d054538
3
- size 4756459616
 
 
 
 
Apollo-34B/model-00008-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e21cf4c4ce58214124afa25d601176b70f7c848386f60658fb575e062a22adf2
3
- size 4756459616
 
 
 
 
Apollo-34B/model-00009-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8ecec2b5ad5eb4d4cbc5e44bf5188114eddceb5b1af158a66e6033d495d63a1c
3
- size 4991369984
 
 
 
 
Apollo-34B/model-00010-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f7fce80f49621847ba8b98ef582a3c0dab11d2e48080ad760cc915b244a8f791
3
- size 4756459616
 
 
 
 
Apollo-34B/model-00011-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a3a931104d8559373f4433513de8256c032cd393ffc67527cd5ba245a7bd1509
3
- size 4756459616
 
 
 
 
Apollo-34B/model-00012-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0a0497a09cfdfe4ee13e75c1832faebd3857e7b212bba060922bb6ebe8dd4612
3
- size 4991369984
 
 
 
 
Apollo-34B/model-00013-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a2dedb6573b997335d2b029ce50e8f578e250741cd1b2a3387643c5f206e2a8d
3
- size 4756459616
 
 
 
 
Apollo-34B/model-00014-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:002f6944b5a48794ee3444639522e5e5462c994150034137f17ee09ad6bce312
3
- size 4756459616
 
 
 
 
Apollo-34B/model-00015-of-00015.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:72be879c20f09c7f1dafe4d62b184bbfeca48f25b4b4417997b658587c990450
3
- size 1211148816
 
 
 
 
Apollo-34B/model.safetensors.index.json DELETED
@@ -1,550 +0,0 @@
1
- {
2
- "metadata": {
3
- "total_size": 68777834496
4
- },
5
- "weight_map": {
6
- "lm_head.weight": "model-00015-of-00015.safetensors",
7
- "model.embed_tokens.weight": "model-00001-of-00015.safetensors",
8
- "model.layers.0.ln1.weight": "model-00001-of-00015.safetensors",
9
- "model.layers.0.ln2.weight": "model-00001-of-00015.safetensors",
10
- "model.layers.0.mlp.down_proj.weight": "model-00001-of-00015.safetensors",
11
- "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00015.safetensors",
12
- "model.layers.0.mlp.up_proj.weight": "model-00001-of-00015.safetensors",
13
- "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00015.safetensors",
14
- "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00015.safetensors",
15
- "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00015.safetensors",
16
- "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00015.safetensors",
17
- "model.layers.1.ln1.weight": "model-00001-of-00015.safetensors",
18
- "model.layers.1.ln2.weight": "model-00001-of-00015.safetensors",
19
- "model.layers.1.mlp.down_proj.weight": "model-00001-of-00015.safetensors",
20
- "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00015.safetensors",
21
- "model.layers.1.mlp.up_proj.weight": "model-00001-of-00015.safetensors",
22
- "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00015.safetensors",
23
- "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00015.safetensors",
24
- "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00015.safetensors",
25
- "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00015.safetensors",
26
- "model.layers.10.ln1.weight": "model-00003-of-00015.safetensors",
27
- "model.layers.10.ln2.weight": "model-00003-of-00015.safetensors",
28
- "model.layers.10.mlp.down_proj.weight": "model-00003-of-00015.safetensors",
29
- "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00015.safetensors",
30
- "model.layers.10.mlp.up_proj.weight": "model-00003-of-00015.safetensors",
31
- "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00015.safetensors",
32
- "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00015.safetensors",
33
- "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00015.safetensors",
34
- "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00015.safetensors",
35
- "model.layers.11.ln1.weight": "model-00003-of-00015.safetensors",
36
- "model.layers.11.ln2.weight": "model-00003-of-00015.safetensors",
37
- "model.layers.11.mlp.down_proj.weight": "model-00003-of-00015.safetensors",
38
- "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00015.safetensors",
39
- "model.layers.11.mlp.up_proj.weight": "model-00003-of-00015.safetensors",
40
- "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00015.safetensors",
41
- "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00015.safetensors",
42
- "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00015.safetensors",
43
- "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00015.safetensors",
44
- "model.layers.12.ln1.weight": "model-00004-of-00015.safetensors",
45
- "model.layers.12.ln2.weight": "model-00004-of-00015.safetensors",
46
- "model.layers.12.mlp.down_proj.weight": "model-00004-of-00015.safetensors",
47
- "model.layers.12.mlp.gate_proj.weight": "model-00004-of-00015.safetensors",
48
- "model.layers.12.mlp.up_proj.weight": "model-00004-of-00015.safetensors",
49
- "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00015.safetensors",
50
- "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00015.safetensors",
51
- "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00015.safetensors",
52
- "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00015.safetensors",
53
- "model.layers.13.ln1.weight": "model-00004-of-00015.safetensors",
54
- "model.layers.13.ln2.weight": "model-00004-of-00015.safetensors",
55
- "model.layers.13.mlp.down_proj.weight": "model-00004-of-00015.safetensors",
56
- "model.layers.13.mlp.gate_proj.weight": "model-00004-of-00015.safetensors",
57
- "model.layers.13.mlp.up_proj.weight": "model-00004-of-00015.safetensors",
58
- "model.layers.13.self_attn.k_proj.weight": "model-00004-of-00015.safetensors",
59
- "model.layers.13.self_attn.o_proj.weight": "model-00004-of-00015.safetensors",
60
- "model.layers.13.self_attn.q_proj.weight": "model-00004-of-00015.safetensors",
61
- "model.layers.13.self_attn.v_proj.weight": "model-00004-of-00015.safetensors",
62
- "model.layers.14.ln1.weight": "model-00004-of-00015.safetensors",
63
- "model.layers.14.ln2.weight": "model-00004-of-00015.safetensors",
64
- "model.layers.14.mlp.down_proj.weight": "model-00004-of-00015.safetensors",
65
- "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00015.safetensors",
66
- "model.layers.14.mlp.up_proj.weight": "model-00004-of-00015.safetensors",
67
- "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00015.safetensors",
68
- "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00015.safetensors",
69
- "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00015.safetensors",
70
- "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00015.safetensors",
71
- "model.layers.15.ln1.weight": "model-00004-of-00015.safetensors",
72
- "model.layers.15.ln2.weight": "model-00004-of-00015.safetensors",
73
- "model.layers.15.mlp.down_proj.weight": "model-00004-of-00015.safetensors",
74
- "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00015.safetensors",
75
- "model.layers.15.mlp.up_proj.weight": "model-00004-of-00015.safetensors",
76
- "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00015.safetensors",
77
- "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00015.safetensors",
78
- "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00015.safetensors",
79
- "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00015.safetensors",
80
- "model.layers.16.ln1.weight": "model-00005-of-00015.safetensors",
81
- "model.layers.16.ln2.weight": "model-00005-of-00015.safetensors",
82
- "model.layers.16.mlp.down_proj.weight": "model-00005-of-00015.safetensors",
83
- "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00015.safetensors",
84
- "model.layers.16.mlp.up_proj.weight": "model-00005-of-00015.safetensors",
85
- "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00015.safetensors",
86
- "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00015.safetensors",
87
- "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00015.safetensors",
88
- "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00015.safetensors",
89
- "model.layers.17.ln1.weight": "model-00005-of-00015.safetensors",
90
- "model.layers.17.ln2.weight": "model-00005-of-00015.safetensors",
91
- "model.layers.17.mlp.down_proj.weight": "model-00005-of-00015.safetensors",
92
- "model.layers.17.mlp.gate_proj.weight": "model-00005-of-00015.safetensors",
93
- "model.layers.17.mlp.up_proj.weight": "model-00005-of-00015.safetensors",
94
- "model.layers.17.self_attn.k_proj.weight": "model-00005-of-00015.safetensors",
95
- "model.layers.17.self_attn.o_proj.weight": "model-00005-of-00015.safetensors",
96
- "model.layers.17.self_attn.q_proj.weight": "model-00005-of-00015.safetensors",
97
- "model.layers.17.self_attn.v_proj.weight": "model-00005-of-00015.safetensors",
98
- "model.layers.18.ln1.weight": "model-00005-of-00015.safetensors",
99
- "model.layers.18.ln2.weight": "model-00005-of-00015.safetensors",
100
- "model.layers.18.mlp.down_proj.weight": "model-00005-of-00015.safetensors",
101
- "model.layers.18.mlp.gate_proj.weight": "model-00005-of-00015.safetensors",
102
- "model.layers.18.mlp.up_proj.weight": "model-00005-of-00015.safetensors",
103
- "model.layers.18.self_attn.k_proj.weight": "model-00005-of-00015.safetensors",
104
- "model.layers.18.self_attn.o_proj.weight": "model-00005-of-00015.safetensors",
105
- "model.layers.18.self_attn.q_proj.weight": "model-00005-of-00015.safetensors",
106
- "model.layers.18.self_attn.v_proj.weight": "model-00005-of-00015.safetensors",
107
- "model.layers.19.ln1.weight": "model-00005-of-00015.safetensors",
108
- "model.layers.19.ln2.weight": "model-00005-of-00015.safetensors",
109
- "model.layers.19.mlp.down_proj.weight": "model-00005-of-00015.safetensors",
110
- "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00015.safetensors",
111
- "model.layers.19.mlp.up_proj.weight": "model-00005-of-00015.safetensors",
112
- "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00015.safetensors",
113
- "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00015.safetensors",
114
- "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00015.safetensors",
115
- "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00015.safetensors",
116
- "model.layers.2.ln1.weight": "model-00001-of-00015.safetensors",
117
- "model.layers.2.ln2.weight": "model-00001-of-00015.safetensors",
118
- "model.layers.2.mlp.down_proj.weight": "model-00001-of-00015.safetensors",
119
- "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00015.safetensors",
120
- "model.layers.2.mlp.up_proj.weight": "model-00001-of-00015.safetensors",
121
- "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00015.safetensors",
122
- "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00015.safetensors",
123
- "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00015.safetensors",
124
- "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00015.safetensors",
125
- "model.layers.20.ln1.weight": "model-00006-of-00015.safetensors",
126
- "model.layers.20.ln2.weight": "model-00006-of-00015.safetensors",
127
- "model.layers.20.mlp.down_proj.weight": "model-00005-of-00015.safetensors",
128
- "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00015.safetensors",
129
- "model.layers.20.mlp.up_proj.weight": "model-00006-of-00015.safetensors",
130
- "model.layers.20.self_attn.k_proj.weight": "model-00005-of-00015.safetensors",
131
- "model.layers.20.self_attn.o_proj.weight": "model-00005-of-00015.safetensors",
132
- "model.layers.20.self_attn.q_proj.weight": "model-00005-of-00015.safetensors",
133
- "model.layers.20.self_attn.v_proj.weight": "model-00005-of-00015.safetensors",
134
- "model.layers.21.ln1.weight": "model-00006-of-00015.safetensors",
135
- "model.layers.21.ln2.weight": "model-00006-of-00015.safetensors",
136
- "model.layers.21.mlp.down_proj.weight": "model-00006-of-00015.safetensors",
137
- "model.layers.21.mlp.gate_proj.weight": "model-00006-of-00015.safetensors",
138
- "model.layers.21.mlp.up_proj.weight": "model-00006-of-00015.safetensors",
139
- "model.layers.21.self_attn.k_proj.weight": "model-00006-of-00015.safetensors",
140
- "model.layers.21.self_attn.o_proj.weight": "model-00006-of-00015.safetensors",
141
- "model.layers.21.self_attn.q_proj.weight": "model-00006-of-00015.safetensors",
142
- "model.layers.21.self_attn.v_proj.weight": "model-00006-of-00015.safetensors",
143
- "model.layers.22.ln1.weight": "model-00006-of-00015.safetensors",
144
- "model.layers.22.ln2.weight": "model-00006-of-00015.safetensors",
145
- "model.layers.22.mlp.down_proj.weight": "model-00006-of-00015.safetensors",
146
- "model.layers.22.mlp.gate_proj.weight": "model-00006-of-00015.safetensors",
147
- "model.layers.22.mlp.up_proj.weight": "model-00006-of-00015.safetensors",
148
- "model.layers.22.self_attn.k_proj.weight": "model-00006-of-00015.safetensors",
149
- "model.layers.22.self_attn.o_proj.weight": "model-00006-of-00015.safetensors",
150
- "model.layers.22.self_attn.q_proj.weight": "model-00006-of-00015.safetensors",
151
- "model.layers.22.self_attn.v_proj.weight": "model-00006-of-00015.safetensors",
152
- "model.layers.23.ln1.weight": "model-00006-of-00015.safetensors",
153
- "model.layers.23.ln2.weight": "model-00006-of-00015.safetensors",
154
- "model.layers.23.mlp.down_proj.weight": "model-00006-of-00015.safetensors",
155
- "model.layers.23.mlp.gate_proj.weight": "model-00006-of-00015.safetensors",
156
- "model.layers.23.mlp.up_proj.weight": "model-00006-of-00015.safetensors",
157
- "model.layers.23.self_attn.k_proj.weight": "model-00006-of-00015.safetensors",
158
- "model.layers.23.self_attn.o_proj.weight": "model-00006-of-00015.safetensors",
159
- "model.layers.23.self_attn.q_proj.weight": "model-00006-of-00015.safetensors",
160
- "model.layers.23.self_attn.v_proj.weight": "model-00006-of-00015.safetensors",
161
- "model.layers.24.ln1.weight": "model-00006-of-00015.safetensors",
162
- "model.layers.24.ln2.weight": "model-00006-of-00015.safetensors",
163
- "model.layers.24.mlp.down_proj.weight": "model-00006-of-00015.safetensors",
164
- "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00015.safetensors",
165
- "model.layers.24.mlp.up_proj.weight": "model-00006-of-00015.safetensors",
166
- "model.layers.24.self_attn.k_proj.weight": "model-00006-of-00015.safetensors",
167
- "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00015.safetensors",
168
- "model.layers.24.self_attn.q_proj.weight": "model-00006-of-00015.safetensors",
169
- "model.layers.24.self_attn.v_proj.weight": "model-00006-of-00015.safetensors",
170
- "model.layers.25.ln1.weight": "model-00007-of-00015.safetensors",
171
- "model.layers.25.ln2.weight": "model-00007-of-00015.safetensors",
172
- "model.layers.25.mlp.down_proj.weight": "model-00007-of-00015.safetensors",
173
- "model.layers.25.mlp.gate_proj.weight": "model-00007-of-00015.safetensors",
174
- "model.layers.25.mlp.up_proj.weight": "model-00007-of-00015.safetensors",
175
- "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00015.safetensors",
176
- "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00015.safetensors",
177
- "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00015.safetensors",
178
- "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00015.safetensors",
179
- "model.layers.26.ln1.weight": "model-00007-of-00015.safetensors",
180
- "model.layers.26.ln2.weight": "model-00007-of-00015.safetensors",
181
- "model.layers.26.mlp.down_proj.weight": "model-00007-of-00015.safetensors",
182
- "model.layers.26.mlp.gate_proj.weight": "model-00007-of-00015.safetensors",
183
- "model.layers.26.mlp.up_proj.weight": "model-00007-of-00015.safetensors",
184
- "model.layers.26.self_attn.k_proj.weight": "model-00007-of-00015.safetensors",
185
- "model.layers.26.self_attn.o_proj.weight": "model-00007-of-00015.safetensors",
186
- "model.layers.26.self_attn.q_proj.weight": "model-00007-of-00015.safetensors",
187
- "model.layers.26.self_attn.v_proj.weight": "model-00007-of-00015.safetensors",
188
- "model.layers.27.ln1.weight": "model-00007-of-00015.safetensors",
189
- "model.layers.27.ln2.weight": "model-00007-of-00015.safetensors",
190
- "model.layers.27.mlp.down_proj.weight": "model-00007-of-00015.safetensors",
191
- "model.layers.27.mlp.gate_proj.weight": "model-00007-of-00015.safetensors",
192
- "model.layers.27.mlp.up_proj.weight": "model-00007-of-00015.safetensors",
193
- "model.layers.27.self_attn.k_proj.weight": "model-00007-of-00015.safetensors",
194
- "model.layers.27.self_attn.o_proj.weight": "model-00007-of-00015.safetensors",
195
- "model.layers.27.self_attn.q_proj.weight": "model-00007-of-00015.safetensors",
196
- "model.layers.27.self_attn.v_proj.weight": "model-00007-of-00015.safetensors",
197
- "model.layers.28.ln1.weight": "model-00007-of-00015.safetensors",
198
- "model.layers.28.ln2.weight": "model-00007-of-00015.safetensors",
199
- "model.layers.28.mlp.down_proj.weight": "model-00007-of-00015.safetensors",
200
- "model.layers.28.mlp.gate_proj.weight": "model-00007-of-00015.safetensors",
201
- "model.layers.28.mlp.up_proj.weight": "model-00007-of-00015.safetensors",
202
- "model.layers.28.self_attn.k_proj.weight": "model-00007-of-00015.safetensors",
203
- "model.layers.28.self_attn.o_proj.weight": "model-00007-of-00015.safetensors",
204
- "model.layers.28.self_attn.q_proj.weight": "model-00007-of-00015.safetensors",
205
- "model.layers.28.self_attn.v_proj.weight": "model-00007-of-00015.safetensors",
206
- "model.layers.29.ln1.weight": "model-00008-of-00015.safetensors",
207
- "model.layers.29.ln2.weight": "model-00008-of-00015.safetensors",
208
- "model.layers.29.mlp.down_proj.weight": "model-00008-of-00015.safetensors",
209
- "model.layers.29.mlp.gate_proj.weight": "model-00007-of-00015.safetensors",
210
- "model.layers.29.mlp.up_proj.weight": "model-00008-of-00015.safetensors",
211
- "model.layers.29.self_attn.k_proj.weight": "model-00007-of-00015.safetensors",
212
- "model.layers.29.self_attn.o_proj.weight": "model-00007-of-00015.safetensors",
213
- "model.layers.29.self_attn.q_proj.weight": "model-00007-of-00015.safetensors",
214
- "model.layers.29.self_attn.v_proj.weight": "model-00007-of-00015.safetensors",
215
- "model.layers.3.ln1.weight": "model-00002-of-00015.safetensors",
216
- "model.layers.3.ln2.weight": "model-00002-of-00015.safetensors",
217
- "model.layers.3.mlp.down_proj.weight": "model-00002-of-00015.safetensors",
218
- "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00015.safetensors",
219
- "model.layers.3.mlp.up_proj.weight": "model-00002-of-00015.safetensors",
220
- "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00015.safetensors",
221
- "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00015.safetensors",
222
- "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00015.safetensors",
223
- "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00015.safetensors",
224
- "model.layers.30.ln1.weight": "model-00008-of-00015.safetensors",
225
- "model.layers.30.ln2.weight": "model-00008-of-00015.safetensors",
226
- "model.layers.30.mlp.down_proj.weight": "model-00008-of-00015.safetensors",
227
- "model.layers.30.mlp.gate_proj.weight": "model-00008-of-00015.safetensors",
228
- "model.layers.30.mlp.up_proj.weight": "model-00008-of-00015.safetensors",
229
- "model.layers.30.self_attn.k_proj.weight": "model-00008-of-00015.safetensors",
230
- "model.layers.30.self_attn.o_proj.weight": "model-00008-of-00015.safetensors",
231
- "model.layers.30.self_attn.q_proj.weight": "model-00008-of-00015.safetensors",
232
- "model.layers.30.self_attn.v_proj.weight": "model-00008-of-00015.safetensors",
233
- "model.layers.31.ln1.weight": "model-00008-of-00015.safetensors",
234
- "model.layers.31.ln2.weight": "model-00008-of-00015.safetensors",
235
- "model.layers.31.mlp.down_proj.weight": "model-00008-of-00015.safetensors",
236
- "model.layers.31.mlp.gate_proj.weight": "model-00008-of-00015.safetensors",
237
- "model.layers.31.mlp.up_proj.weight": "model-00008-of-00015.safetensors",
238
- "model.layers.31.self_attn.k_proj.weight": "model-00008-of-00015.safetensors",
239
- "model.layers.31.self_attn.o_proj.weight": "model-00008-of-00015.safetensors",
240
- "model.layers.31.self_attn.q_proj.weight": "model-00008-of-00015.safetensors",
241
- "model.layers.31.self_attn.v_proj.weight": "model-00008-of-00015.safetensors",
242
- "model.layers.32.ln1.weight": "model-00008-of-00015.safetensors",
243
- "model.layers.32.ln2.weight": "model-00008-of-00015.safetensors",
244
- "model.layers.32.mlp.down_proj.weight": "model-00008-of-00015.safetensors",
245
- "model.layers.32.mlp.gate_proj.weight": "model-00008-of-00015.safetensors",
246
- "model.layers.32.mlp.up_proj.weight": "model-00008-of-00015.safetensors",
247
- "model.layers.32.self_attn.k_proj.weight": "model-00008-of-00015.safetensors",
248
- "model.layers.32.self_attn.o_proj.weight": "model-00008-of-00015.safetensors",
249
- "model.layers.32.self_attn.q_proj.weight": "model-00008-of-00015.safetensors",
250
- "model.layers.32.self_attn.v_proj.weight": "model-00008-of-00015.safetensors",
251
- "model.layers.33.ln1.weight": "model-00009-of-00015.safetensors",
252
- "model.layers.33.ln2.weight": "model-00009-of-00015.safetensors",
253
- "model.layers.33.mlp.down_proj.weight": "model-00008-of-00015.safetensors",
254
- "model.layers.33.mlp.gate_proj.weight": "model-00008-of-00015.safetensors",
255
- "model.layers.33.mlp.up_proj.weight": "model-00009-of-00015.safetensors",
256
- "model.layers.33.self_attn.k_proj.weight": "model-00008-of-00015.safetensors",
257
- "model.layers.33.self_attn.o_proj.weight": "model-00008-of-00015.safetensors",
258
- "model.layers.33.self_attn.q_proj.weight": "model-00008-of-00015.safetensors",
259
- "model.layers.33.self_attn.v_proj.weight": "model-00008-of-00015.safetensors",
260
- "model.layers.34.ln1.weight": "model-00009-of-00015.safetensors",
261
- "model.layers.34.ln2.weight": "model-00009-of-00015.safetensors",
262
- "model.layers.34.mlp.down_proj.weight": "model-00009-of-00015.safetensors",
263
- "model.layers.34.mlp.gate_proj.weight": "model-00009-of-00015.safetensors",
264
- "model.layers.34.mlp.up_proj.weight": "model-00009-of-00015.safetensors",
265
- "model.layers.34.self_attn.k_proj.weight": "model-00009-of-00015.safetensors",
266
- "model.layers.34.self_attn.o_proj.weight": "model-00009-of-00015.safetensors",
267
- "model.layers.34.self_attn.q_proj.weight": "model-00009-of-00015.safetensors",
268
- "model.layers.34.self_attn.v_proj.weight": "model-00009-of-00015.safetensors",
269
- "model.layers.35.ln1.weight": "model-00009-of-00015.safetensors",
270
- "model.layers.35.ln2.weight": "model-00009-of-00015.safetensors",
271
- "model.layers.35.mlp.down_proj.weight": "model-00009-of-00015.safetensors",
272
- "model.layers.35.mlp.gate_proj.weight": "model-00009-of-00015.safetensors",
273
- "model.layers.35.mlp.up_proj.weight": "model-00009-of-00015.safetensors",
274
- "model.layers.35.self_attn.k_proj.weight": "model-00009-of-00015.safetensors",
275
- "model.layers.35.self_attn.o_proj.weight": "model-00009-of-00015.safetensors",
276
- "model.layers.35.self_attn.q_proj.weight": "model-00009-of-00015.safetensors",
277
- "model.layers.35.self_attn.v_proj.weight": "model-00009-of-00015.safetensors",
278
- "model.layers.36.ln1.weight": "model-00009-of-00015.safetensors",
279
- "model.layers.36.ln2.weight": "model-00009-of-00015.safetensors",
280
- "model.layers.36.mlp.down_proj.weight": "model-00009-of-00015.safetensors",
281
- "model.layers.36.mlp.gate_proj.weight": "model-00009-of-00015.safetensors",
282
- "model.layers.36.mlp.up_proj.weight": "model-00009-of-00015.safetensors",
283
- "model.layers.36.self_attn.k_proj.weight": "model-00009-of-00015.safetensors",
284
- "model.layers.36.self_attn.o_proj.weight": "model-00009-of-00015.safetensors",
285
- "model.layers.36.self_attn.q_proj.weight": "model-00009-of-00015.safetensors",
286
- "model.layers.36.self_attn.v_proj.weight": "model-00009-of-00015.safetensors",
287
- "model.layers.37.ln1.weight": "model-00009-of-00015.safetensors",
288
- "model.layers.37.ln2.weight": "model-00009-of-00015.safetensors",
289
- "model.layers.37.mlp.down_proj.weight": "model-00009-of-00015.safetensors",
290
- "model.layers.37.mlp.gate_proj.weight": "model-00009-of-00015.safetensors",
291
- "model.layers.37.mlp.up_proj.weight": "model-00009-of-00015.safetensors",
292
- "model.layers.37.self_attn.k_proj.weight": "model-00009-of-00015.safetensors",
293
- "model.layers.37.self_attn.o_proj.weight": "model-00009-of-00015.safetensors",
294
- "model.layers.37.self_attn.q_proj.weight": "model-00009-of-00015.safetensors",
295
- "model.layers.37.self_attn.v_proj.weight": "model-00009-of-00015.safetensors",
296
- "model.layers.38.ln1.weight": "model-00010-of-00015.safetensors",
297
- "model.layers.38.ln2.weight": "model-00010-of-00015.safetensors",
298
- "model.layers.38.mlp.down_proj.weight": "model-00010-of-00015.safetensors",
299
- "model.layers.38.mlp.gate_proj.weight": "model-00010-of-00015.safetensors",
300
- "model.layers.38.mlp.up_proj.weight": "model-00010-of-00015.safetensors",
301
- "model.layers.38.self_attn.k_proj.weight": "model-00009-of-00015.safetensors",
302
- "model.layers.38.self_attn.o_proj.weight": "model-00009-of-00015.safetensors",
303
- "model.layers.38.self_attn.q_proj.weight": "model-00009-of-00015.safetensors",
304
- "model.layers.38.self_attn.v_proj.weight": "model-00009-of-00015.safetensors",
305
- "model.layers.39.ln1.weight": "model-00010-of-00015.safetensors",
306
- "model.layers.39.ln2.weight": "model-00010-of-00015.safetensors",
307
- "model.layers.39.mlp.down_proj.weight": "model-00010-of-00015.safetensors",
308
- "model.layers.39.mlp.gate_proj.weight": "model-00010-of-00015.safetensors",
309
- "model.layers.39.mlp.up_proj.weight": "model-00010-of-00015.safetensors",
310
- "model.layers.39.self_attn.k_proj.weight": "model-00010-of-00015.safetensors",
311
- "model.layers.39.self_attn.o_proj.weight": "model-00010-of-00015.safetensors",
312
- "model.layers.39.self_attn.q_proj.weight": "model-00010-of-00015.safetensors",
313
- "model.layers.39.self_attn.v_proj.weight": "model-00010-of-00015.safetensors",
314
- "model.layers.4.ln1.weight": "model-00002-of-00015.safetensors",
315
- "model.layers.4.ln2.weight": "model-00002-of-00015.safetensors",
316
- "model.layers.4.mlp.down_proj.weight": "model-00002-of-00015.safetensors",
317
- "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00015.safetensors",
318
- "model.layers.4.mlp.up_proj.weight": "model-00002-of-00015.safetensors",
319
- "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00015.safetensors",
320
- "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00015.safetensors",
321
- "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00015.safetensors",
322
- "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00015.safetensors",
323
- "model.layers.40.ln1.weight": "model-00010-of-00015.safetensors",
324
- "model.layers.40.ln2.weight": "model-00010-of-00015.safetensors",
325
- "model.layers.40.mlp.down_proj.weight": "model-00010-of-00015.safetensors",
326
- "model.layers.40.mlp.gate_proj.weight": "model-00010-of-00015.safetensors",
327
- "model.layers.40.mlp.up_proj.weight": "model-00010-of-00015.safetensors",
328
- "model.layers.40.self_attn.k_proj.weight": "model-00010-of-00015.safetensors",
329
- "model.layers.40.self_attn.o_proj.weight": "model-00010-of-00015.safetensors",
330
- "model.layers.40.self_attn.q_proj.weight": "model-00010-of-00015.safetensors",
331
- "model.layers.40.self_attn.v_proj.weight": "model-00010-of-00015.safetensors",
332
- "model.layers.41.ln1.weight": "model-00010-of-00015.safetensors",
333
- "model.layers.41.ln2.weight": "model-00010-of-00015.safetensors",
334
- "model.layers.41.mlp.down_proj.weight": "model-00010-of-00015.safetensors",
335
- "model.layers.41.mlp.gate_proj.weight": "model-00010-of-00015.safetensors",
336
- "model.layers.41.mlp.up_proj.weight": "model-00010-of-00015.safetensors",
337
- "model.layers.41.self_attn.k_proj.weight": "model-00010-of-00015.safetensors",
338
- "model.layers.41.self_attn.o_proj.weight": "model-00010-of-00015.safetensors",
339
- "model.layers.41.self_attn.q_proj.weight": "model-00010-of-00015.safetensors",
340
- "model.layers.41.self_attn.v_proj.weight": "model-00010-of-00015.safetensors",
341
- "model.layers.42.ln1.weight": "model-00011-of-00015.safetensors",
342
- "model.layers.42.ln2.weight": "model-00011-of-00015.safetensors",
343
- "model.layers.42.mlp.down_proj.weight": "model-00011-of-00015.safetensors",
344
- "model.layers.42.mlp.gate_proj.weight": "model-00010-of-00015.safetensors",
345
- "model.layers.42.mlp.up_proj.weight": "model-00011-of-00015.safetensors",
346
- "model.layers.42.self_attn.k_proj.weight": "model-00010-of-00015.safetensors",
347
- "model.layers.42.self_attn.o_proj.weight": "model-00010-of-00015.safetensors",
348
- "model.layers.42.self_attn.q_proj.weight": "model-00010-of-00015.safetensors",
349
- "model.layers.42.self_attn.v_proj.weight": "model-00010-of-00015.safetensors",
350
- "model.layers.43.ln1.weight": "model-00011-of-00015.safetensors",
351
- "model.layers.43.ln2.weight": "model-00011-of-00015.safetensors",
352
- "model.layers.43.mlp.down_proj.weight": "model-00011-of-00015.safetensors",
353
- "model.layers.43.mlp.gate_proj.weight": "model-00011-of-00015.safetensors",
354
- "model.layers.43.mlp.up_proj.weight": "model-00011-of-00015.safetensors",
355
- "model.layers.43.self_attn.k_proj.weight": "model-00011-of-00015.safetensors",
356
- "model.layers.43.self_attn.o_proj.weight": "model-00011-of-00015.safetensors",
357
- "model.layers.43.self_attn.q_proj.weight": "model-00011-of-00015.safetensors",
358
- "model.layers.43.self_attn.v_proj.weight": "model-00011-of-00015.safetensors",
359
- "model.layers.44.ln1.weight": "model-00011-of-00015.safetensors",
360
- "model.layers.44.ln2.weight": "model-00011-of-00015.safetensors",
361
- "model.layers.44.mlp.down_proj.weight": "model-00011-of-00015.safetensors",
362
- "model.layers.44.mlp.gate_proj.weight": "model-00011-of-00015.safetensors",
363
- "model.layers.44.mlp.up_proj.weight": "model-00011-of-00015.safetensors",
364
- "model.layers.44.self_attn.k_proj.weight": "model-00011-of-00015.safetensors",
365
- "model.layers.44.self_attn.o_proj.weight": "model-00011-of-00015.safetensors",
366
- "model.layers.44.self_attn.q_proj.weight": "model-00011-of-00015.safetensors",
367
- "model.layers.44.self_attn.v_proj.weight": "model-00011-of-00015.safetensors",
368
- "model.layers.45.ln1.weight": "model-00011-of-00015.safetensors",
369
- "model.layers.45.ln2.weight": "model-00011-of-00015.safetensors",
370
- "model.layers.45.mlp.down_proj.weight": "model-00011-of-00015.safetensors",
371
- "model.layers.45.mlp.gate_proj.weight": "model-00011-of-00015.safetensors",
372
- "model.layers.45.mlp.up_proj.weight": "model-00011-of-00015.safetensors",
373
- "model.layers.45.self_attn.k_proj.weight": "model-00011-of-00015.safetensors",
374
- "model.layers.45.self_attn.o_proj.weight": "model-00011-of-00015.safetensors",
375
- "model.layers.45.self_attn.q_proj.weight": "model-00011-of-00015.safetensors",
376
- "model.layers.45.self_attn.v_proj.weight": "model-00011-of-00015.safetensors",
377
- "model.layers.46.ln1.weight": "model-00012-of-00015.safetensors",
378
- "model.layers.46.ln2.weight": "model-00012-of-00015.safetensors",
379
- "model.layers.46.mlp.down_proj.weight": "model-00011-of-00015.safetensors",
380
- "model.layers.46.mlp.gate_proj.weight": "model-00011-of-00015.safetensors",
381
- "model.layers.46.mlp.up_proj.weight": "model-00012-of-00015.safetensors",
382
- "model.layers.46.self_attn.k_proj.weight": "model-00011-of-00015.safetensors",
383
- "model.layers.46.self_attn.o_proj.weight": "model-00011-of-00015.safetensors",
384
- "model.layers.46.self_attn.q_proj.weight": "model-00011-of-00015.safetensors",
385
- "model.layers.46.self_attn.v_proj.weight": "model-00011-of-00015.safetensors",
386
- "model.layers.47.ln1.weight": "model-00012-of-00015.safetensors",
387
- "model.layers.47.ln2.weight": "model-00012-of-00015.safetensors",
388
- "model.layers.47.mlp.down_proj.weight": "model-00012-of-00015.safetensors",
389
- "model.layers.47.mlp.gate_proj.weight": "model-00012-of-00015.safetensors",
390
- "model.layers.47.mlp.up_proj.weight": "model-00012-of-00015.safetensors",
391
- "model.layers.47.self_attn.k_proj.weight": "model-00012-of-00015.safetensors",
392
- "model.layers.47.self_attn.o_proj.weight": "model-00012-of-00015.safetensors",
393
- "model.layers.47.self_attn.q_proj.weight": "model-00012-of-00015.safetensors",
394
- "model.layers.47.self_attn.v_proj.weight": "model-00012-of-00015.safetensors",
395
- "model.layers.48.ln1.weight": "model-00012-of-00015.safetensors",
396
- "model.layers.48.ln2.weight": "model-00012-of-00015.safetensors",
397
- "model.layers.48.mlp.down_proj.weight": "model-00012-of-00015.safetensors",
398
- "model.layers.48.mlp.gate_proj.weight": "model-00012-of-00015.safetensors",
399
- "model.layers.48.mlp.up_proj.weight": "model-00012-of-00015.safetensors",
400
- "model.layers.48.self_attn.k_proj.weight": "model-00012-of-00015.safetensors",
401
- "model.layers.48.self_attn.o_proj.weight": "model-00012-of-00015.safetensors",
402
- "model.layers.48.self_attn.q_proj.weight": "model-00012-of-00015.safetensors",
403
- "model.layers.48.self_attn.v_proj.weight": "model-00012-of-00015.safetensors",
404
- "model.layers.49.ln1.weight": "model-00012-of-00015.safetensors",
405
- "model.layers.49.ln2.weight": "model-00012-of-00015.safetensors",
406
- "model.layers.49.mlp.down_proj.weight": "model-00012-of-00015.safetensors",
407
- "model.layers.49.mlp.gate_proj.weight": "model-00012-of-00015.safetensors",
408
- "model.layers.49.mlp.up_proj.weight": "model-00012-of-00015.safetensors",
409
- "model.layers.49.self_attn.k_proj.weight": "model-00012-of-00015.safetensors",
410
- "model.layers.49.self_attn.o_proj.weight": "model-00012-of-00015.safetensors",
411
- "model.layers.49.self_attn.q_proj.weight": "model-00012-of-00015.safetensors",
412
- "model.layers.49.self_attn.v_proj.weight": "model-00012-of-00015.safetensors",
413
- "model.layers.5.ln1.weight": "model-00002-of-00015.safetensors",
414
- "model.layers.5.ln2.weight": "model-00002-of-00015.safetensors",
415
- "model.layers.5.mlp.down_proj.weight": "model-00002-of-00015.safetensors",
416
- "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00015.safetensors",
417
- "model.layers.5.mlp.up_proj.weight": "model-00002-of-00015.safetensors",
418
- "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00015.safetensors",
419
- "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00015.safetensors",
420
- "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00015.safetensors",
421
- "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00015.safetensors",
422
- "model.layers.50.ln1.weight": "model-00012-of-00015.safetensors",
423
- "model.layers.50.ln2.weight": "model-00012-of-00015.safetensors",
424
- "model.layers.50.mlp.down_proj.weight": "model-00012-of-00015.safetensors",
425
- "model.layers.50.mlp.gate_proj.weight": "model-00012-of-00015.safetensors",
426
- "model.layers.50.mlp.up_proj.weight": "model-00012-of-00015.safetensors",
427
- "model.layers.50.self_attn.k_proj.weight": "model-00012-of-00015.safetensors",
428
- "model.layers.50.self_attn.o_proj.weight": "model-00012-of-00015.safetensors",
429
- "model.layers.50.self_attn.q_proj.weight": "model-00012-of-00015.safetensors",
430
- "model.layers.50.self_attn.v_proj.weight": "model-00012-of-00015.safetensors",
431
- "model.layers.51.ln1.weight": "model-00013-of-00015.safetensors",
432
- "model.layers.51.ln2.weight": "model-00013-of-00015.safetensors",
433
- "model.layers.51.mlp.down_proj.weight": "model-00013-of-00015.safetensors",
434
- "model.layers.51.mlp.gate_proj.weight": "model-00013-of-00015.safetensors",
435
- "model.layers.51.mlp.up_proj.weight": "model-00013-of-00015.safetensors",
436
- "model.layers.51.self_attn.k_proj.weight": "model-00012-of-00015.safetensors",
437
- "model.layers.51.self_attn.o_proj.weight": "model-00012-of-00015.safetensors",
438
- "model.layers.51.self_attn.q_proj.weight": "model-00012-of-00015.safetensors",
439
- "model.layers.51.self_attn.v_proj.weight": "model-00012-of-00015.safetensors",
440
- "model.layers.52.ln1.weight": "model-00013-of-00015.safetensors",
441
- "model.layers.52.ln2.weight": "model-00013-of-00015.safetensors",
442
- "model.layers.52.mlp.down_proj.weight": "model-00013-of-00015.safetensors",
443
- "model.layers.52.mlp.gate_proj.weight": "model-00013-of-00015.safetensors",
444
- "model.layers.52.mlp.up_proj.weight": "model-00013-of-00015.safetensors",
445
- "model.layers.52.self_attn.k_proj.weight": "model-00013-of-00015.safetensors",
446
- "model.layers.52.self_attn.o_proj.weight": "model-00013-of-00015.safetensors",
447
- "model.layers.52.self_attn.q_proj.weight": "model-00013-of-00015.safetensors",
448
- "model.layers.52.self_attn.v_proj.weight": "model-00013-of-00015.safetensors",
449
- "model.layers.53.ln1.weight": "model-00013-of-00015.safetensors",
450
- "model.layers.53.ln2.weight": "model-00013-of-00015.safetensors",
451
- "model.layers.53.mlp.down_proj.weight": "model-00013-of-00015.safetensors",
452
- "model.layers.53.mlp.gate_proj.weight": "model-00013-of-00015.safetensors",
453
- "model.layers.53.mlp.up_proj.weight": "model-00013-of-00015.safetensors",
454
- "model.layers.53.self_attn.k_proj.weight": "model-00013-of-00015.safetensors",
455
- "model.layers.53.self_attn.o_proj.weight": "model-00013-of-00015.safetensors",
456
- "model.layers.53.self_attn.q_proj.weight": "model-00013-of-00015.safetensors",
457
- "model.layers.53.self_attn.v_proj.weight": "model-00013-of-00015.safetensors",
458
- "model.layers.54.ln1.weight": "model-00013-of-00015.safetensors",
459
- "model.layers.54.ln2.weight": "model-00013-of-00015.safetensors",
460
- "model.layers.54.mlp.down_proj.weight": "model-00013-of-00015.safetensors",
461
- "model.layers.54.mlp.gate_proj.weight": "model-00013-of-00015.safetensors",
462
- "model.layers.54.mlp.up_proj.weight": "model-00013-of-00015.safetensors",
463
- "model.layers.54.self_attn.k_proj.weight": "model-00013-of-00015.safetensors",
464
- "model.layers.54.self_attn.o_proj.weight": "model-00013-of-00015.safetensors",
465
- "model.layers.54.self_attn.q_proj.weight": "model-00013-of-00015.safetensors",
466
- "model.layers.54.self_attn.v_proj.weight": "model-00013-of-00015.safetensors",
467
- "model.layers.55.ln1.weight": "model-00014-of-00015.safetensors",
468
- "model.layers.55.ln2.weight": "model-00014-of-00015.safetensors",
469
- "model.layers.55.mlp.down_proj.weight": "model-00014-of-00015.safetensors",
470
- "model.layers.55.mlp.gate_proj.weight": "model-00013-of-00015.safetensors",
471
- "model.layers.55.mlp.up_proj.weight": "model-00014-of-00015.safetensors",
472
- "model.layers.55.self_attn.k_proj.weight": "model-00013-of-00015.safetensors",
473
- "model.layers.55.self_attn.o_proj.weight": "model-00013-of-00015.safetensors",
474
- "model.layers.55.self_attn.q_proj.weight": "model-00013-of-00015.safetensors",
475
- "model.layers.55.self_attn.v_proj.weight": "model-00013-of-00015.safetensors",
476
- "model.layers.56.ln1.weight": "model-00014-of-00015.safetensors",
477
- "model.layers.56.ln2.weight": "model-00014-of-00015.safetensors",
478
- "model.layers.56.mlp.down_proj.weight": "model-00014-of-00015.safetensors",
479
- "model.layers.56.mlp.gate_proj.weight": "model-00014-of-00015.safetensors",
480
- "model.layers.56.mlp.up_proj.weight": "model-00014-of-00015.safetensors",
481
- "model.layers.56.self_attn.k_proj.weight": "model-00014-of-00015.safetensors",
482
- "model.layers.56.self_attn.o_proj.weight": "model-00014-of-00015.safetensors",
483
- "model.layers.56.self_attn.q_proj.weight": "model-00014-of-00015.safetensors",
484
- "model.layers.56.self_attn.v_proj.weight": "model-00014-of-00015.safetensors",
485
- "model.layers.57.ln1.weight": "model-00014-of-00015.safetensors",
486
- "model.layers.57.ln2.weight": "model-00014-of-00015.safetensors",
487
- "model.layers.57.mlp.down_proj.weight": "model-00014-of-00015.safetensors",
488
- "model.layers.57.mlp.gate_proj.weight": "model-00014-of-00015.safetensors",
489
- "model.layers.57.mlp.up_proj.weight": "model-00014-of-00015.safetensors",
490
- "model.layers.57.self_attn.k_proj.weight": "model-00014-of-00015.safetensors",
491
- "model.layers.57.self_attn.o_proj.weight": "model-00014-of-00015.safetensors",
492
- "model.layers.57.self_attn.q_proj.weight": "model-00014-of-00015.safetensors",
493
- "model.layers.57.self_attn.v_proj.weight": "model-00014-of-00015.safetensors",
494
- "model.layers.58.ln1.weight": "model-00014-of-00015.safetensors",
495
- "model.layers.58.ln2.weight": "model-00014-of-00015.safetensors",
496
- "model.layers.58.mlp.down_proj.weight": "model-00014-of-00015.safetensors",
497
- "model.layers.58.mlp.gate_proj.weight": "model-00014-of-00015.safetensors",
498
- "model.layers.58.mlp.up_proj.weight": "model-00014-of-00015.safetensors",
499
- "model.layers.58.self_attn.k_proj.weight": "model-00014-of-00015.safetensors",
500
- "model.layers.58.self_attn.o_proj.weight": "model-00014-of-00015.safetensors",
501
- "model.layers.58.self_attn.q_proj.weight": "model-00014-of-00015.safetensors",
502
- "model.layers.58.self_attn.v_proj.weight": "model-00014-of-00015.safetensors",
503
- "model.layers.59.ln1.weight": "model-00015-of-00015.safetensors",
504
- "model.layers.59.ln2.weight": "model-00015-of-00015.safetensors",
505
- "model.layers.59.mlp.down_proj.weight": "model-00014-of-00015.safetensors",
506
- "model.layers.59.mlp.gate_proj.weight": "model-00014-of-00015.safetensors",
507
- "model.layers.59.mlp.up_proj.weight": "model-00015-of-00015.safetensors",
508
- "model.layers.59.self_attn.k_proj.weight": "model-00014-of-00015.safetensors",
509
- "model.layers.59.self_attn.o_proj.weight": "model-00014-of-00015.safetensors",
510
- "model.layers.59.self_attn.q_proj.weight": "model-00014-of-00015.safetensors",
511
- "model.layers.59.self_attn.v_proj.weight": "model-00014-of-00015.safetensors",
512
- "model.layers.6.ln1.weight": "model-00002-of-00015.safetensors",
513
- "model.layers.6.ln2.weight": "model-00002-of-00015.safetensors",
514
- "model.layers.6.mlp.down_proj.weight": "model-00002-of-00015.safetensors",
515
- "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00015.safetensors",
516
- "model.layers.6.mlp.up_proj.weight": "model-00002-of-00015.safetensors",
517
- "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00015.safetensors",
518
- "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00015.safetensors",
519
- "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00015.safetensors",
520
- "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00015.safetensors",
521
- "model.layers.7.ln1.weight": "model-00003-of-00015.safetensors",
522
- "model.layers.7.ln2.weight": "model-00003-of-00015.safetensors",
523
- "model.layers.7.mlp.down_proj.weight": "model-00002-of-00015.safetensors",
524
- "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00015.safetensors",
525
- "model.layers.7.mlp.up_proj.weight": "model-00003-of-00015.safetensors",
526
- "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00015.safetensors",
527
- "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00015.safetensors",
528
- "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00015.safetensors",
529
- "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00015.safetensors",
530
- "model.layers.8.ln1.weight": "model-00003-of-00015.safetensors",
531
- "model.layers.8.ln2.weight": "model-00003-of-00015.safetensors",
532
- "model.layers.8.mlp.down_proj.weight": "model-00003-of-00015.safetensors",
533
- "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00015.safetensors",
534
- "model.layers.8.mlp.up_proj.weight": "model-00003-of-00015.safetensors",
535
- "model.layers.8.self_attn.k_proj.weight": "model-00003-of-00015.safetensors",
536
- "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00015.safetensors",
537
- "model.layers.8.self_attn.q_proj.weight": "model-00003-of-00015.safetensors",
538
- "model.layers.8.self_attn.v_proj.weight": "model-00003-of-00015.safetensors",
539
- "model.layers.9.ln1.weight": "model-00003-of-00015.safetensors",
540
- "model.layers.9.ln2.weight": "model-00003-of-00015.safetensors",
541
- "model.layers.9.mlp.down_proj.weight": "model-00003-of-00015.safetensors",
542
- "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00015.safetensors",
543
- "model.layers.9.mlp.up_proj.weight": "model-00003-of-00015.safetensors",
544
- "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00015.safetensors",
545
- "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00015.safetensors",
546
- "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00015.safetensors",
547
- "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00015.safetensors",
548
- "model.norm.weight": "model-00015-of-00015.safetensors"
549
- }
550
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
Apollo-34B/modeling_yi.py DELETED
@@ -1,1035 +0,0 @@
1
- """ PyTorch Yi model."""
2
- import math
3
- from typing import List, Optional, Tuple, Union
4
-
5
- import torch.utils.checkpoint
6
- from einops import repeat
7
- from packaging import version
8
- from torch import nn
9
- from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
10
- from transformers.activations import ACT2FN
11
- from transformers.modeling_outputs import (
12
- BaseModelOutputWithPast,
13
- CausalLMOutputWithPast,
14
- SequenceClassifierOutputWithPast,
15
- )
16
- from transformers.modeling_utils import PreTrainedModel
17
- from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
18
- from transformers.utils import (
19
- add_start_docstrings,
20
- add_start_docstrings_to_model_forward,
21
- logging,
22
- replace_return_docstrings,
23
- )
24
-
25
- from .configuration_yi import YiConfig
26
-
27
- is_flash_attn_available = True
28
- try:
29
- from flash_attn import flash_attn_func, __version__
30
-
31
- assert version.parse(__version__) >= version.parse(
32
- "2.3.0"
33
- ), "please update your flash_attn version (>= 2.3.0)"
34
- except ModuleNotFoundError:
35
- is_flash_attn_available = False
36
-
37
- logger = logging.get_logger(__name__)
38
-
39
- _CONFIG_FOR_DOC = "YiConfig"
40
-
41
-
42
- # Copied from transformers.models.bart.modeling_bart._make_causal_mask
43
- def _make_causal_mask(
44
- input_ids_shape: torch.Size,
45
- dtype: torch.dtype,
46
- device: torch.device,
47
- past_key_values_length: int = 0,
48
- ):
49
- """
50
- Make causal mask used for bi-directional self-attention.
51
- """
52
- bsz, tgt_len = input_ids_shape
53
- mask = torch.full(
54
- (tgt_len, tgt_len),
55
- torch.tensor(torch.finfo(dtype).min, device=device),
56
- device=device,
57
- )
58
- mask_cond = torch.arange(mask.size(-1), device=device)
59
- mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
60
- mask = mask.to(dtype)
61
-
62
- if past_key_values_length > 0:
63
- mask = torch.cat(
64
- [
65
- torch.zeros(
66
- tgt_len, past_key_values_length, dtype=dtype, device=device
67
- ),
68
- mask,
69
- ],
70
- dim=-1,
71
- )
72
- return mask[None, None, :, :].expand(
73
- bsz, 1, tgt_len, tgt_len + past_key_values_length
74
- )
75
-
76
-
77
- # Copied from transformers.models.bart.modeling_bart._expand_mask
78
- def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
79
- """
80
- Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
81
- """
82
- bsz, src_len = mask.size()
83
- tgt_len = tgt_len if tgt_len is not None else src_len
84
-
85
- expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
86
-
87
- inverted_mask = 1.0 - expanded_mask
88
-
89
- return inverted_mask.masked_fill(
90
- inverted_mask.to(torch.bool), torch.finfo(dtype).min
91
- )
92
-
93
-
94
- class YiRMSNorm(nn.Module):
95
- def __init__(self, hidden_size, eps=1e-5):
96
- """
97
- YiRMSNorm is equivalent to T5LayerNorm
98
- """
99
- super().__init__()
100
- self.weight = nn.Parameter(torch.ones(hidden_size))
101
- self.variance_epsilon = eps
102
-
103
- def forward(self, hidden_states):
104
- input_dtype = hidden_states.dtype
105
- hidden_states = hidden_states.to(torch.float32)
106
- variance = hidden_states.pow(2).mean(-1, keepdim=True)
107
- hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
108
-
109
- return self.weight * hidden_states.to(input_dtype)
110
-
111
-
112
- ALL_LAYERNORM_LAYERS.append(YiRMSNorm)
113
-
114
-
115
- class YiRotaryEmbedding(torch.nn.Module):
116
- def __init__(self, dim, max_position_embeddings=4096, base=5000000, device=None):
117
- super().__init__()
118
-
119
- self.dim = dim
120
- self.max_position_embeddings = max_position_embeddings
121
- self.base = base
122
-
123
- # Build here to make `torch.jit.trace` work.
124
- self._set_cos_sin_cache(seq_len=max_position_embeddings, device=device)
125
-
126
- def _set_cos_sin_cache(self, seq_len, device):
127
- self.max_seq_len_cached = seq_len
128
- inv_freq = 1.0 / (
129
- self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)
130
- )
131
- t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.float32)
132
- freqs = torch.einsum("i,j->ij", t, inv_freq)
133
- # Different from paper, but it uses a different permutation in order to obtain the same calculation
134
- emb = torch.cat((freqs, freqs), dim=-1)
135
- self.register_buffer(
136
- "cos_cached", emb.cos()[None, None, :, :], persistent=False
137
- )
138
- self.register_buffer(
139
- "sin_cached", emb.sin()[None, None, :, :], persistent=False
140
- )
141
-
142
- def forward(self, x, seq_len=None):
143
- # x: [bs, num_attention_heads, seq_len, head_size]
144
- if seq_len > self.max_seq_len_cached:
145
- self._set_cos_sin_cache(seq_len=seq_len, device=x.device)
146
-
147
- return (
148
- self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
149
- self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
150
- )
151
-
152
-
153
- def rotate_half(x):
154
- """Rotates half the hidden dims of the input."""
155
- x1 = x[..., : x.shape[-1] // 2]
156
- x2 = x[..., x.shape[-1] // 2 :]
157
- return torch.cat((-x2, x1), dim=-1)
158
-
159
-
160
- def apply_rotary_pos_emb(q, k, cos, sin, position_ids, flash_attn_available):
161
- # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
162
- cos = cos.squeeze(1).squeeze(0) # [seq_len, dim]
163
- sin = sin.squeeze(1).squeeze(0) # [seq_len, dim]
164
- expand_dim = 2 if flash_attn_available else 1
165
- cos = cos[position_ids].unsqueeze(expand_dim) # [bs, seq_len, dim]
166
- sin = sin[position_ids].unsqueeze(expand_dim) # [bs, seq_len, dim]
167
- q_embed = (q * cos) + (rotate_half(q) * sin)
168
- k_embed = (k * cos) + (rotate_half(k) * sin)
169
- return q_embed, k_embed
170
-
171
-
172
- class YiMLP(nn.Module):
173
- def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str):
174
- super().__init__()
175
- self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
176
- self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
177
- self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
178
- self.act_fn = ACT2FN[hidden_act]
179
-
180
- def forward(self, x):
181
- return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
182
-
183
-
184
- class YiAttention(nn.Module):
185
- """Multi-headed attention from 'Attention Is All You Need' paper"""
186
-
187
- def __init__(self, config: YiConfig):
188
- super().__init__()
189
- self.config = config
190
- self.hidden_size = config.hidden_size
191
- self.num_heads = config.num_attention_heads
192
- self.head_dim = self.hidden_size // self.num_heads
193
- self.num_key_value_heads = config.num_key_value_heads
194
- self.num_key_value_groups = self.num_heads // self.num_key_value_heads
195
- self.max_position_embeddings = config.max_position_embeddings
196
-
197
- if (self.head_dim * self.num_heads) != self.hidden_size:
198
- raise ValueError(
199
- f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
200
- f" and `num_heads`: {self.num_heads})."
201
- )
202
- self.q_proj = nn.Linear(
203
- self.hidden_size, self.num_heads * self.head_dim, bias=False
204
- )
205
- self.k_proj = nn.Linear(
206
- self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False
207
- )
208
- self.v_proj = nn.Linear(
209
- self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False
210
- )
211
- self.o_proj = nn.Linear(
212
- self.num_heads * self.head_dim, self.hidden_size, bias=False
213
- )
214
-
215
- self.rotary_emb = YiRotaryEmbedding(
216
- self.head_dim,
217
- max_position_embeddings=self.max_position_embeddings,
218
- base=self.config.rope_theta,
219
- )
220
-
221
- def forward(
222
- self,
223
- hidden_states: torch.Tensor,
224
- attention_mask: Optional[torch.Tensor] = None,
225
- position_ids: Optional[torch.LongTensor] = None,
226
- past_key_value: Optional[Tuple[torch.Tensor]] = None,
227
- output_attentions: bool = False,
228
- use_cache: bool = False,
229
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
230
- bsz, q_len, _ = hidden_states.size()
231
-
232
- query_states = self.q_proj(hidden_states).view(
233
- bsz, q_len, self.num_heads, self.head_dim
234
- )
235
-
236
- key_states = self.k_proj(hidden_states).view(
237
- bsz, q_len, self.num_key_value_heads, self.head_dim
238
- )
239
- value_states = self.v_proj(hidden_states).view(
240
- bsz, q_len, self.num_key_value_heads, self.head_dim
241
- )
242
-
243
- if not is_flash_attn_available:
244
- if self.num_key_value_groups > 1:
245
- key_states = repeat(
246
- key_states, f"b n h d -> b n (h {self.num_key_value_groups}) d"
247
- )
248
- value_states = repeat(
249
- value_states, f"b n h d -> b n (h {self.num_key_value_groups}) d"
250
- )
251
-
252
- # b n h d -> b h n d
253
- query_states = query_states.transpose(1, 2)
254
- key_states = key_states.transpose(1, 2)
255
- value_states = value_states.transpose(1, 2)
256
-
257
- seq_dim = 1 if is_flash_attn_available else 2
258
- kv_seq_len = key_states.shape[seq_dim]
259
- if past_key_value is not None:
260
- kv_seq_len += past_key_value[0].shape[seq_dim]
261
- cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
262
- query_states, key_states = apply_rotary_pos_emb(
263
- query_states, key_states, cos, sin, position_ids, is_flash_attn_available
264
- )
265
-
266
- if past_key_value is not None:
267
- # reuse k, v, self_attention
268
- key_states = torch.cat([past_key_value[0], key_states], dim=seq_dim)
269
- value_states = torch.cat([past_key_value[1], value_states], dim=seq_dim)
270
-
271
- past_key_value = (key_states, value_states) if use_cache else None
272
-
273
- if is_flash_attn_available:
274
- attn_output = flash_attn_func(
275
- query_states, key_states, value_states, dropout_p=0.0, causal=True
276
- )
277
- else:
278
- attn_weights = torch.matmul(
279
- query_states, key_states.transpose(2, 3)
280
- ) / math.sqrt(self.head_dim)
281
-
282
- if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
283
- raise ValueError(
284
- f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
285
- f" {attn_weights.size()}"
286
- )
287
-
288
- if attention_mask is not None:
289
- if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
290
- raise ValueError(
291
- f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is"
292
- f"{attention_mask.size()}"
293
- )
294
- attn_weights = attn_weights + attention_mask
295
- dtype_min = torch.tensor(
296
- torch.finfo(attn_weights.dtype).min,
297
- device=attn_weights.device,
298
- dtype=attn_weights.dtype,
299
- )
300
- attn_weights = torch.max(attn_weights, dtype_min)
301
-
302
- # upcast attention to fp32
303
- attn_weights = nn.functional.softmax(
304
- attn_weights, dim=-1, dtype=torch.float32
305
- ).to(query_states.dtype)
306
- attn_output = torch.matmul(attn_weights, value_states)
307
-
308
- if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
309
- raise ValueError(
310
- f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
311
- f" {attn_output.size()}"
312
- )
313
-
314
- if not is_flash_attn_available:
315
- attn_output = attn_output.transpose(1, 2)
316
-
317
- attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
318
-
319
- attn_output = self.o_proj(attn_output)
320
-
321
- if not output_attentions:
322
- attn_weights = None
323
-
324
- return attn_output, attn_weights, past_key_value
325
-
326
-
327
- class YiDecoderLayer(nn.Module):
328
- def __init__(self, config: YiConfig):
329
- super().__init__()
330
-
331
- self.hidden_size = config.hidden_size
332
- self.self_attn = YiAttention(config=config)
333
- self.mlp = YiMLP(
334
- hidden_size=self.hidden_size,
335
- intermediate_size=config.intermediate_size,
336
- hidden_act=config.hidden_act,
337
- )
338
-
339
- self.ln1 = YiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
340
- self.ln2 = YiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
341
-
342
- def forward(
343
- self,
344
- hidden_states: torch.Tensor,
345
- attention_mask: Optional[torch.Tensor] = None,
346
- position_ids: Optional[torch.LongTensor] = None,
347
- past_key_value: Optional[Tuple[torch.Tensor]] = None,
348
- output_attentions: Optional[bool] = False,
349
- use_cache: Optional[bool] = False,
350
- ) -> Tuple[
351
- torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
352
- ]:
353
- """
354
- Args:
355
- hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
356
- attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
357
- `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
358
- output_attentions (`bool`, *optional*):
359
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under
360
- returned tensors for more detail.
361
- use_cache (`bool`, *optional*):
362
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
363
- (see `past_key_values`).
364
- past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
365
- """
366
-
367
- residual = hidden_states
368
-
369
- hidden_states = self.ln1(hidden_states)
370
-
371
- # Self Attention
372
- hidden_states, self_attn_weights, present_key_value = self.self_attn(
373
- hidden_states=hidden_states,
374
- attention_mask=attention_mask,
375
- position_ids=position_ids,
376
- past_key_value=past_key_value,
377
- output_attentions=output_attentions,
378
- use_cache=use_cache,
379
- )
380
- hidden_states = residual + hidden_states
381
-
382
- # Fully Connected
383
- residual = hidden_states
384
- hidden_states = self.ln2(hidden_states)
385
- hidden_states = self.mlp(hidden_states)
386
- hidden_states = residual + hidden_states
387
-
388
- outputs = (hidden_states,)
389
-
390
- if output_attentions:
391
- outputs += (self_attn_weights,)
392
-
393
- if use_cache:
394
- outputs += (present_key_value,)
395
-
396
- return outputs
397
-
398
-
399
- Yi_START_DOCSTRING = r"""
400
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
401
- library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
402
- etc.)
403
-
404
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
405
- Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
406
- and behavior.
407
-
408
- Parameters:
409
- config ([`YiConfig`]):
410
- Model configuration class with all the parameters of the model. Initializing with a config file does not
411
- load the weights associated with the model, only the configuration. Check out the
412
- [`~PreTrainedModel.from_pretrained`] method to load the model weights.
413
- """
414
-
415
-
416
- @add_start_docstrings(
417
- "The bare Yi Model outputting raw hidden-states without any specific head on top.",
418
- Yi_START_DOCSTRING,
419
- )
420
- class YiPreTrainedModel(PreTrainedModel):
421
- config_class = YiConfig
422
- base_model_prefix = "model"
423
- supports_gradient_checkpointing = True
424
- _no_split_modules = ["YiDecoderLayer"]
425
- _skip_keys_device_placement = "past_key_values"
426
-
427
- def _init_weights(self, module):
428
- std = self.config.initializer_range
429
- if isinstance(module, nn.Linear):
430
- module.weight.data.normal_(mean=0.0, std=std)
431
- if module.bias is not None:
432
- module.bias.data.zero_()
433
- elif isinstance(module, nn.Embedding):
434
- module.weight.data.normal_(mean=0.0, std=std)
435
- if module.padding_idx is not None:
436
- module.weight.data[module.padding_idx].zero_()
437
-
438
- def _set_gradient_checkpointing(self, module, value=False):
439
- if isinstance(module, YiModel):
440
- module.gradient_checkpointing = value
441
-
442
-
443
- Yi_INPUTS_DOCSTRING = r"""
444
- Args:
445
- input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
446
- Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
447
- it.
448
-
449
- Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
450
- [`PreTrainedTokenizer.__call__`] for details.
451
-
452
- [What are input IDs?](../glossary#input-ids)
453
- attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
454
- Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
455
-
456
- - 1 for tokens that are **not masked**,
457
- - 0 for tokens that are **masked**.
458
-
459
- [What are attention masks?](../glossary#attention-mask)
460
-
461
- Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
462
- [`PreTrainedTokenizer.__call__`] for details.
463
-
464
- If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
465
- `past_key_values`).
466
-
467
- If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
468
- and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
469
- information on the default strategy.
470
-
471
- - 1 indicates the head is **not masked**,
472
- - 0 indicates the head is **masked**.
473
- position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
474
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
475
- config.n_positions - 1]`.
476
-
477
- [What are position IDs?](../glossary#position-ids)
478
- past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
479
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
480
- `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
481
- `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
482
-
483
- Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
484
- blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
485
-
486
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
487
- don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
488
- `decoder_input_ids` of shape `(batch_size, sequence_length)`.
489
- inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
490
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
491
- is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
492
- model's internal embedding lookup matrix.
493
- use_cache (`bool`, *optional*):
494
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
495
- `past_key_values`).
496
- output_attentions (`bool`, *optional*):
497
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
498
- tensors for more detail.
499
- output_hidden_states (`bool`, *optional*):
500
- Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
501
- more detail.
502
- return_dict (`bool`, *optional*):
503
- Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
504
- """
505
-
506
-
507
- @add_start_docstrings(
508
- "The bare Yi Model outputting raw hidden-states without any specific head on top.",
509
- Yi_START_DOCSTRING,
510
- )
511
- class YiModel(YiPreTrainedModel):
512
- """
513
- Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`YiDecoderLayer`]
514
-
515
- Args:
516
- config: YiConfig
517
- """
518
-
519
- def __init__(self, config: YiConfig):
520
- super().__init__(config)
521
- self.padding_idx = config.pad_token_id
522
- self.vocab_size = config.vocab_size
523
-
524
- self.embed_tokens = nn.Embedding(
525
- config.vocab_size, config.hidden_size, self.padding_idx
526
- )
527
- self.layers = nn.ModuleList(
528
- [YiDecoderLayer(config) for _ in range(config.num_hidden_layers)]
529
- )
530
-
531
- self.norm = YiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
532
-
533
- self.gradient_checkpointing = False
534
- # Initialize weights and apply final processing
535
- self.post_init()
536
-
537
- def get_input_embeddings(self):
538
- return self.embed_tokens
539
-
540
- def set_input_embeddings(self, value):
541
- self.embed_tokens = value
542
-
543
- # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
544
- def _prepare_decoder_attention_mask(
545
- self, attention_mask, input_ids, inputs_embeds, past_key_values_length
546
- ):
547
- input_shape = (
548
- input_ids.shape if input_ids is not None else inputs_embeds.shape[:-1]
549
- )
550
- # create causal mask
551
- # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
552
- combined_attention_mask = None
553
- if input_shape[-1] > 1:
554
- combined_attention_mask = _make_causal_mask(
555
- input_shape,
556
- inputs_embeds.dtype,
557
- device=inputs_embeds.device,
558
- past_key_values_length=past_key_values_length,
559
- )
560
-
561
- if attention_mask is not None:
562
- # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
563
- expanded_attn_mask = _expand_mask(
564
- attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
565
- ).to(inputs_embeds.device)
566
- combined_attention_mask = (
567
- expanded_attn_mask
568
- if combined_attention_mask is None
569
- else expanded_attn_mask + combined_attention_mask
570
- )
571
-
572
- return combined_attention_mask
573
-
574
- @add_start_docstrings_to_model_forward(Yi_INPUTS_DOCSTRING)
575
- def forward(
576
- self,
577
- input_ids: torch.LongTensor = None,
578
- attention_mask: Optional[torch.Tensor] = None,
579
- position_ids: Optional[torch.LongTensor] = None,
580
- past_key_values: Optional[List[torch.FloatTensor]] = None,
581
- inputs_embeds: Optional[torch.FloatTensor] = None,
582
- use_cache: Optional[bool] = None,
583
- output_attentions: Optional[bool] = None,
584
- output_hidden_states: Optional[bool] = None,
585
- return_dict: Optional[bool] = None,
586
- ) -> Union[Tuple, BaseModelOutputWithPast]:
587
- output_attentions = (
588
- output_attentions
589
- if output_attentions is not None
590
- else self.config.output_attentions
591
- )
592
- output_hidden_states = (
593
- output_hidden_states
594
- if output_hidden_states is not None
595
- else self.config.output_hidden_states
596
- )
597
- use_cache = use_cache if use_cache is not None else self.config.use_cache
598
-
599
- return_dict = (
600
- return_dict if return_dict is not None else self.config.use_return_dict
601
- )
602
-
603
- # retrieve input_ids and inputs_embeds
604
- if input_ids is not None and inputs_embeds is not None:
605
- raise ValueError(
606
- "You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time"
607
- )
608
- elif input_ids is not None:
609
- batch_size, seq_length = input_ids.shape
610
- elif inputs_embeds is not None:
611
- batch_size, seq_length, _ = inputs_embeds.shape
612
- else:
613
- raise ValueError(
614
- "You have to specify either decoder_input_ids or decoder_inputs_embeds"
615
- )
616
-
617
- seq_length_with_past = seq_length
618
- past_key_values_length = 0
619
-
620
- if past_key_values is not None:
621
- past_key_values_length = past_key_values[0][0].shape[2]
622
- seq_length_with_past = seq_length_with_past + past_key_values_length
623
-
624
- if position_ids is None:
625
- device = input_ids.device if input_ids is not None else inputs_embeds.device
626
- position_ids = torch.arange(
627
- past_key_values_length,
628
- seq_length + past_key_values_length,
629
- dtype=torch.long,
630
- device=device,
631
- )
632
- position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
633
- else:
634
- position_ids = position_ids.view(-1, seq_length).long()
635
-
636
- if inputs_embeds is None:
637
- inputs_embeds = self.embed_tokens(input_ids)
638
-
639
- if not is_flash_attn_available:
640
- # embed positions
641
- if attention_mask is None:
642
- attention_mask = torch.ones(
643
- (batch_size, seq_length_with_past),
644
- dtype=torch.bool,
645
- device=inputs_embeds.device,
646
- )
647
- attention_mask = self._prepare_decoder_attention_mask(
648
- attention_mask,
649
- input_ids,
650
- inputs_embeds,
651
- past_key_values_length,
652
- )
653
- else:
654
- attention_mask = None
655
-
656
- hidden_states = inputs_embeds
657
- if self.gradient_checkpointing and self.training:
658
- if use_cache:
659
- logger.warning_once(
660
- "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
661
- )
662
- use_cache = False
663
-
664
- # decoder layers
665
- all_hidden_states = () if output_hidden_states else None
666
- all_self_attns = () if output_attentions else None
667
- next_decoder_cache = () if use_cache else None
668
-
669
- for idx, decoder_layer in enumerate(self.layers):
670
- if output_hidden_states:
671
- all_hidden_states += (hidden_states,)
672
-
673
- past_key_value = (
674
- past_key_values[idx] if past_key_values is not None else None
675
- )
676
-
677
- if self.gradient_checkpointing and self.training:
678
-
679
- def create_custom_forward(module):
680
- def custom_forward(*inputs):
681
- # None for past_key_value
682
- return module(*inputs, past_key_value, output_attentions)
683
-
684
- return custom_forward
685
-
686
- layer_outputs = torch.utils.checkpoint.checkpoint(
687
- create_custom_forward(decoder_layer),
688
- hidden_states,
689
- attention_mask,
690
- position_ids,
691
- )
692
- else:
693
- layer_outputs = decoder_layer(
694
- hidden_states,
695
- attention_mask=attention_mask,
696
- position_ids=position_ids,
697
- past_key_value=past_key_value,
698
- output_attentions=output_attentions,
699
- use_cache=use_cache,
700
- )
701
-
702
- hidden_states = layer_outputs[0]
703
-
704
- if use_cache:
705
- next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
706
-
707
- if output_attentions:
708
- all_self_attns += (layer_outputs[1],)
709
-
710
- hidden_states = self.norm(hidden_states)
711
- # add hidden states from the last decoder layer
712
- if output_hidden_states:
713
- all_hidden_states += (hidden_states,)
714
-
715
- next_cache = next_decoder_cache if use_cache else None
716
- if not return_dict:
717
- return tuple(
718
- v
719
- for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
720
- if v is not None
721
- )
722
- return BaseModelOutputWithPast(
723
- last_hidden_state=hidden_states,
724
- past_key_values=next_cache,
725
- hidden_states=all_hidden_states,
726
- attentions=all_self_attns,
727
- )
728
-
729
-
730
- class YiForCausalLM(YiPreTrainedModel):
731
- _tied_weights_keys = ["lm_head.weight"]
732
-
733
- def __init__(self, config):
734
- super().__init__(config)
735
- self.model = YiModel(config)
736
-
737
- self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
738
-
739
- # Initialize weights and apply final processing
740
- self.post_init()
741
-
742
- def get_input_embeddings(self):
743
- return self.model.embed_tokens
744
-
745
- def set_input_embeddings(self, value):
746
- self.model.embed_tokens = value
747
-
748
- def get_output_embeddings(self):
749
- return self.lm_head
750
-
751
- def set_output_embeddings(self, new_embeddings):
752
- self.lm_head = new_embeddings
753
-
754
- def set_decoder(self, decoder):
755
- self.model = decoder
756
-
757
- def get_decoder(self):
758
- return self.model
759
-
760
- @add_start_docstrings_to_model_forward(Yi_INPUTS_DOCSTRING)
761
- @replace_return_docstrings(
762
- output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC
763
- )
764
- def forward(
765
- self,
766
- input_ids: torch.LongTensor = None,
767
- attention_mask: Optional[torch.Tensor] = None,
768
- position_ids: Optional[torch.LongTensor] = None,
769
- past_key_values: Optional[List[torch.FloatTensor]] = None,
770
- inputs_embeds: Optional[torch.FloatTensor] = None,
771
- labels: Optional[torch.LongTensor] = None,
772
- use_cache: Optional[bool] = None,
773
- output_attentions: Optional[bool] = None,
774
- output_hidden_states: Optional[bool] = None,
775
- return_dict: Optional[bool] = None,
776
- ) -> Union[Tuple, CausalLMOutputWithPast]:
777
- r"""
778
- Args:
779
- labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
780
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
781
- config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
782
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
783
-
784
- Returns:
785
-
786
- Example:
787
-
788
- ```python
789
- >>> from transformers import AutoTokenizer, YiForCausalLM
790
-
791
- >>> model = YiForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
792
- >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
793
-
794
- >>> prompt = "Hey, are you conscious? Can you talk to me?"
795
- >>> inputs = tokenizer(prompt, return_tensors="pt")
796
-
797
- >>> # Generate
798
- >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
799
- >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
800
- "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
801
- ```"""
802
-
803
- output_attentions = (
804
- output_attentions
805
- if output_attentions is not None
806
- else self.config.output_attentions
807
- )
808
- output_hidden_states = (
809
- output_hidden_states
810
- if output_hidden_states is not None
811
- else self.config.output_hidden_states
812
- )
813
- return_dict = (
814
- return_dict if return_dict is not None else self.config.use_return_dict
815
- )
816
-
817
- # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
818
- outputs = self.model(
819
- input_ids=input_ids,
820
- attention_mask=attention_mask,
821
- position_ids=position_ids,
822
- past_key_values=past_key_values,
823
- inputs_embeds=inputs_embeds,
824
- use_cache=use_cache,
825
- output_attentions=output_attentions,
826
- output_hidden_states=output_hidden_states,
827
- return_dict=return_dict,
828
- )
829
-
830
- hidden_states = outputs[0]
831
- logits = self.lm_head(hidden_states)
832
-
833
- loss = None
834
- if labels is not None:
835
- # Shift so that tokens < n predict n
836
- shift_logits = logits[..., :-1, :].contiguous()
837
- shift_labels = labels[..., 1:].contiguous()
838
- # Flatten the tokens
839
- loss_fct = CrossEntropyLoss()
840
- shift_logits = shift_logits.view(-1, self.config.vocab_size)
841
- shift_labels = shift_labels.view(-1)
842
- # Enable model parallelism
843
- shift_labels = shift_labels.to(shift_logits.device)
844
- loss = loss_fct(shift_logits, shift_labels)
845
-
846
- if not return_dict:
847
- output = (logits,) + outputs[1:]
848
- return (loss,) + output if loss is not None else output
849
-
850
- return CausalLMOutputWithPast(
851
- loss=loss,
852
- logits=logits,
853
- past_key_values=outputs.past_key_values,
854
- hidden_states=outputs.hidden_states,
855
- attentions=outputs.attentions,
856
- )
857
-
858
- def prepare_inputs_for_generation(
859
- self,
860
- input_ids,
861
- past_key_values=None,
862
- attention_mask=None,
863
- inputs_embeds=None,
864
- **kwargs,
865
- ):
866
- if past_key_values:
867
- input_ids = input_ids[:, -1:]
868
-
869
- position_ids = kwargs.get("position_ids", None)
870
- if attention_mask is not None and position_ids is None:
871
- # create position_ids on the fly for batch generation
872
- position_ids = attention_mask.long().cumsum(-1) - 1
873
- position_ids.masked_fill_(attention_mask == 0, 1)
874
- if past_key_values:
875
- position_ids = position_ids[:, -1].unsqueeze(-1)
876
-
877
- # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
878
- if inputs_embeds is not None and past_key_values is None:
879
- model_inputs = {"inputs_embeds": inputs_embeds}
880
- else:
881
- model_inputs = {"input_ids": input_ids}
882
-
883
- model_inputs.update(
884
- {
885
- "position_ids": position_ids,
886
- "past_key_values": past_key_values,
887
- "use_cache": kwargs.get("use_cache"),
888
- "attention_mask": attention_mask,
889
- }
890
- )
891
- return model_inputs
892
-
893
- @staticmethod
894
- def _reorder_cache(past_key_values, beam_idx):
895
- reordered_past = ()
896
- for layer_past in past_key_values:
897
- reordered_past += (
898
- tuple(
899
- past_state.index_select(0, beam_idx.to(past_state.device))
900
- for past_state in layer_past
901
- ),
902
- )
903
- return reordered_past
904
-
905
-
906
- @add_start_docstrings(
907
- """
908
- The Yi Model transformer with a sequence classification head on top (linear layer).
909
-
910
- [`YiForSequenceClassification`] uses the last token in order to do the classification, as other causal models
911
- (e.g. GPT-2) do.
912
-
913
- Since it does classification on the last token, it requires to know the position of the last token. If a
914
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
915
- no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
916
- padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
917
- each row of the batch).
918
- """,
919
- Yi_START_DOCSTRING,
920
- )
921
- class YiForSequenceClassification(YiPreTrainedModel):
922
- def __init__(self, config):
923
- super().__init__(config)
924
- self.num_labels = config.num_labels
925
- self.model = YiModel(config)
926
- self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
927
-
928
- # Initialize weights and apply final processing
929
- self.post_init()
930
-
931
- def get_input_embeddings(self):
932
- return self.model.embed_tokens
933
-
934
- def set_input_embeddings(self, value):
935
- self.model.embed_tokens = value
936
-
937
- @add_start_docstrings_to_model_forward(Yi_INPUTS_DOCSTRING)
938
- def forward(
939
- self,
940
- input_ids: torch.LongTensor = None,
941
- attention_mask: Optional[torch.Tensor] = None,
942
- position_ids: Optional[torch.LongTensor] = None,
943
- past_key_values: Optional[List[torch.FloatTensor]] = None,
944
- inputs_embeds: Optional[torch.FloatTensor] = None,
945
- labels: Optional[torch.LongTensor] = None,
946
- use_cache: Optional[bool] = None,
947
- output_attentions: Optional[bool] = None,
948
- output_hidden_states: Optional[bool] = None,
949
- return_dict: Optional[bool] = None,
950
- ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
951
- r"""
952
- labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
953
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
954
- config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
955
- `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
956
- """
957
- return_dict = (
958
- return_dict if return_dict is not None else self.config.use_return_dict
959
- )
960
-
961
- transformer_outputs = self.model(
962
- input_ids,
963
- attention_mask=attention_mask,
964
- position_ids=position_ids,
965
- past_key_values=past_key_values,
966
- inputs_embeds=inputs_embeds,
967
- use_cache=use_cache,
968
- output_attentions=output_attentions,
969
- output_hidden_states=output_hidden_states,
970
- return_dict=return_dict,
971
- )
972
- hidden_states = transformer_outputs[0]
973
- logits = self.score(hidden_states)
974
-
975
- if input_ids is not None:
976
- batch_size = input_ids.shape[0]
977
- else:
978
- batch_size = inputs_embeds.shape[0]
979
-
980
- if self.config.pad_token_id is None and batch_size != 1:
981
- raise ValueError(
982
- "Cannot handle batch sizes > 1 if no padding token is defined."
983
- )
984
- if self.config.pad_token_id is None:
985
- sequence_lengths = -1
986
- else:
987
- if input_ids is not None:
988
- sequence_lengths = (
989
- torch.eq(input_ids, self.config.pad_token_id).long().argmax(-1) - 1
990
- ).to(logits.device)
991
- else:
992
- sequence_lengths = -1
993
-
994
- pooled_logits = logits[
995
- torch.arange(batch_size, device=logits.device), sequence_lengths
996
- ]
997
-
998
- loss = None
999
- if labels is not None:
1000
- labels = labels.to(logits.device)
1001
- if self.config.problem_type is None:
1002
- if self.num_labels == 1:
1003
- self.config.problem_type = "regression"
1004
- elif self.num_labels > 1 and (
1005
- labels.dtype == torch.long or labels.dtype == torch.int
1006
- ):
1007
- self.config.problem_type = "single_label_classification"
1008
- else:
1009
- self.config.problem_type = "multi_label_classification"
1010
-
1011
- if self.config.problem_type == "regression":
1012
- loss_fct = MSELoss()
1013
- if self.num_labels == 1:
1014
- loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
1015
- else:
1016
- loss = loss_fct(pooled_logits, labels)
1017
- elif self.config.problem_type == "single_label_classification":
1018
- loss_fct = CrossEntropyLoss()
1019
- loss = loss_fct(
1020
- pooled_logits.view(-1, self.num_labels), labels.view(-1)
1021
- )
1022
- elif self.config.problem_type == "multi_label_classification":
1023
- loss_fct = BCEWithLogitsLoss()
1024
- loss = loss_fct(pooled_logits, labels)
1025
- if not return_dict:
1026
- output = (pooled_logits,) + transformer_outputs[1:]
1027
- return ((loss,) + output) if loss is not None else output
1028
-
1029
- return SequenceClassifierOutputWithPast(
1030
- loss=loss,
1031
- logits=pooled_logits,
1032
- past_key_values=transformer_outputs.past_key_values,
1033
- hidden_states=transformer_outputs.hidden_states,
1034
- attentions=transformer_outputs.attentions,
1035
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
Apollo-34B/tokenization_yi.py DELETED
@@ -1,255 +0,0 @@
1
- import os
2
- from shutil import copyfile
3
- from typing import Any, Dict, List, Optional, Tuple
4
-
5
- import sentencepiece as spm
6
- from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
7
- from transformers.utils import logging
8
-
9
- logger = logging.get_logger(__name__)
10
-
11
- VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
12
-
13
- PRETRAINED_VOCAB_FILES_MAP = {
14
- "vocab_file": {},
15
- "tokenizer_file": {},
16
- }
17
- PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
18
-
19
-
20
- class YiTokenizer(PreTrainedTokenizer):
21
- """
22
- Construct a Yi tokenizer. Based on byte-level Byte-Pair-Encoding.
23
-
24
- Args:
25
- vocab_file (`str`):
26
- Path to the vocabulary file.
27
- """
28
-
29
- vocab_files_names = VOCAB_FILES_NAMES
30
- pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
31
- max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
32
- model_input_names = ["input_ids", "attention_mask"]
33
-
34
- def __init__(
35
- self,
36
- vocab_file,
37
- unk_token="<unk>",
38
- bos_token="<|startoftext|>",
39
- eos_token="<|endoftext|>",
40
- pad_token="<unk>",
41
- sp_model_kwargs: Optional[Dict[str, Any]] = None,
42
- add_bos_token=True,
43
- add_eos_token=False,
44
- clean_up_tokenization_spaces=False,
45
- **kwargs,
46
- ):
47
- self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
48
- bos_token = (
49
- AddedToken(bos_token, lstrip=False, rstrip=False)
50
- if isinstance(bos_token, str)
51
- else bos_token
52
- )
53
- eos_token = (
54
- AddedToken(eos_token, lstrip=False, rstrip=False)
55
- if isinstance(eos_token, str)
56
- else eos_token
57
- )
58
- unk_token = (
59
- AddedToken(unk_token, lstrip=False, rstrip=False)
60
- if isinstance(unk_token, str)
61
- else unk_token
62
- )
63
- pad_token = (
64
- AddedToken(pad_token, lstrip=False, rstrip=False)
65
- if isinstance(pad_token, str)
66
- else pad_token
67
- )
68
- self.vocab_file = vocab_file
69
- self.add_bos_token = add_bos_token
70
- self.add_eos_token = add_eos_token
71
- self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
72
- self.sp_model.Load(vocab_file)
73
- super().__init__(
74
- bos_token=bos_token,
75
- eos_token=eos_token,
76
- unk_token=unk_token,
77
- pad_token=pad_token,
78
- add_bos_token=add_bos_token,
79
- add_eos_token=add_eos_token,
80
- sp_model_kwargs=self.sp_model_kwargs,
81
- clean_up_tokenization_spaces=clean_up_tokenization_spaces,
82
- **kwargs,
83
- )
84
-
85
- def __getstate__(self):
86
- state = self.__dict__.copy()
87
- state["sp_model"] = None
88
- return state
89
-
90
- def __setstate__(self, d):
91
- self.__dict__ = d
92
- self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
93
- self.sp_model.Load(self.vocab_file)
94
-
95
- @property
96
- def vocab_size(self):
97
- """Returns vocab size"""
98
- return self.sp_model.get_piece_size()
99
-
100
- def get_vocab(self):
101
- """Returns vocab as a dict"""
102
- vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
103
- vocab.update(self.added_tokens_encoder)
104
- return vocab
105
-
106
- def _tokenize(self, text):
107
- """Returns a tokenized string."""
108
- return self.sp_model.encode(text, out_type=str)
109
-
110
- def _convert_token_to_id(self, token):
111
- """Converts a token (str) in an id using the vocab."""
112
- return self.sp_model.piece_to_id(token)
113
-
114
- def _convert_id_to_token(self, index):
115
- """Converts an index (integer) in a token (str) using the vocab."""
116
- token = self.sp_model.IdToPiece(index)
117
- return token
118
-
119
- def convert_tokens_to_string(self, tokens):
120
- """Converts a sequence of tokens (string) in a single string."""
121
- current_sub_tokens = []
122
- out_string = ""
123
- prev_is_special = False
124
- for i, token in enumerate(tokens):
125
- # make sure that special tokens are not decoded using sentencepiece model
126
- if token in self.all_special_tokens:
127
- if not prev_is_special and i != 0:
128
- out_string += " "
129
- out_string += self.sp_model.decode(current_sub_tokens) + token
130
- prev_is_special = True
131
- current_sub_tokens = []
132
- else:
133
- current_sub_tokens.append(token)
134
- prev_is_special = False
135
- out_string += self.sp_model.decode(current_sub_tokens)
136
- return out_string
137
-
138
- def save_vocabulary(
139
- self, save_directory, filename_prefix: Optional[str] = None
140
- ) -> Tuple[str]:
141
- """
142
- Save the vocabulary and special tokens file to a directory.
143
-
144
- Args:
145
- save_directory (`str`):
146
- The directory in which to save the vocabulary.
147
-
148
- Returns:
149
- `Tuple(str)`: Paths to the files saved.
150
- """
151
- if not os.path.isdir(save_directory):
152
- logger.error(f"Vocabulary path ({save_directory}) should be a directory")
153
- return
154
- out_vocab_file = os.path.join(
155
- save_directory,
156
- (filename_prefix + "-" if filename_prefix else "")
157
- + VOCAB_FILES_NAMES["vocab_file"],
158
- )
159
-
160
- if os.path.abspath(self.vocab_file) != os.path.abspath(
161
- out_vocab_file
162
- ) and os.path.isfile(self.vocab_file):
163
- copyfile(self.vocab_file, out_vocab_file)
164
- elif not os.path.isfile(self.vocab_file):
165
- with open(out_vocab_file, "wb") as fi:
166
- content_spiece_model = self.sp_model.serialized_model_proto()
167
- fi.write(content_spiece_model)
168
-
169
- return (out_vocab_file,)
170
-
171
- def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
172
- bos_token_id = [self.bos_token_id] if self.add_bos_token else []
173
- eos_token_id = [self.eos_token_id] if self.add_eos_token else []
174
-
175
- output = bos_token_id + token_ids_0 + eos_token_id
176
-
177
- if token_ids_1 is not None:
178
- output = output + bos_token_id + token_ids_1 + eos_token_id
179
-
180
- return output
181
-
182
- def get_special_tokens_mask(
183
- self,
184
- token_ids_0: List[int],
185
- token_ids_1: Optional[List[int]] = None,
186
- already_has_special_tokens: bool = False,
187
- ) -> List[int]:
188
- """
189
- Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
190
- special tokens using the tokenizer `prepare_for_model` method.
191
-
192
- Args:
193
- token_ids_0 (`List[int]`):
194
- List of IDs.
195
- token_ids_1 (`List[int]`, *optional*):
196
- Optional second list of IDs for sequence pairs.
197
- already_has_special_tokens (`bool`, *optional*, defaults to `False`):
198
- Whether or not the token list is already formatted with special tokens for the model.
199
-
200
- Returns:
201
- `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
202
- """
203
- if already_has_special_tokens:
204
- return super().get_special_tokens_mask(
205
- token_ids_0=token_ids_0,
206
- token_ids_1=token_ids_1,
207
- already_has_special_tokens=True,
208
- )
209
-
210
- bos_token_id = [1] if self.add_bos_token else []
211
- eos_token_id = [1] if self.add_eos_token else []
212
-
213
- if token_ids_1 is None:
214
- return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
215
- return (
216
- bos_token_id
217
- + ([0] * len(token_ids_0))
218
- + eos_token_id
219
- + bos_token_id
220
- + ([0] * len(token_ids_1))
221
- + eos_token_id
222
- )
223
-
224
- def create_token_type_ids_from_sequences(
225
- self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
226
- ) -> List[int]:
227
- """
228
- Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
229
- sequence pair mask has the following format:
230
-
231
- ```
232
- 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
233
- | first sequence | second sequence |
234
- ```
235
-
236
- if token_ids_1 is None, only returns the first portion of the mask (0s).
237
-
238
- Args:
239
- token_ids_0 (`List[int]`):
240
- List of ids.
241
- token_ids_1 (`List[int]`, *optional*):
242
- Optional second list of IDs for sequence pairs.
243
-
244
- Returns:
245
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
246
- """
247
- bos_token_id = [self.bos_token_id] if self.add_bos_token else []
248
- eos_token_id = [self.eos_token_id] if self.add_eos_token else []
249
-
250
- output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
251
-
252
- if token_ids_1 is not None:
253
- output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
254
-
255
- return output
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
Apollo-34B/tokenizer.json DELETED
The diff for this file is too large to render. See raw diff
 
Apollo-34B/tokenizer.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:386c49cf943d71aa110361135338c50e38beeff0a66593480421f37b319e1a39
3
- size 1033105
 
 
 
 
Apollo-34B/tokenizer_config.json DELETED
@@ -1,9 +0,0 @@
1
- {
2
- "auto_map": {
3
- "AutoTokenizer": ["tokenization_yi.YiTokenizer", null]
4
- },
5
- "add_bos_token": false,
6
- "add_eos_token": false,
7
- "model_max_length": 4096,
8
- "tokenizer_class": "YiTokenizer"
9
- }