heongjun127 commited on
Commit
7a9614b
·
verified ·
1 Parent(s): dda3411

Delete 7f15baedd46858153d817445aff032f4d6cf4939

Browse files
7f15baedd46858153d817445aff032f4d6cf4939/config.json DELETED
@@ -1,32 +0,0 @@
1
- {
2
- "activation_function": "silu",
3
- "architectures": [
4
- "ExaoneForCausalLM"
5
- ],
6
- "attention_dropout": 0.0,
7
- "auto_map": {
8
- "AutoConfig": "configuration_exaone.ExaoneConfig",
9
- "AutoModelForCausalLM": "modeling_exaone.ExaoneForCausalLM",
10
- "AutoModelForSequenceClassification": "modeling_exaone.ExaoneForSequenceClassification"
11
- },
12
- "bos_token_id": 1,
13
- "embed_dropout": 0.0,
14
- "eos_token_id": 361,
15
- "hidden_size": 4096,
16
- "initializer_range": 0.02,
17
- "intermediate_size": 14336,
18
- "layer_norm_epsilon": 1e-05,
19
- "max_position_embeddings": 4096,
20
- "model_type": "exaone",
21
- "num_attention_heads": 32,
22
- "num_key_value_heads": 8,
23
- "num_layers": 32,
24
- "pad_token_id": 0,
25
- "rope_scaling": null,
26
- "rope_theta": 500000.0,
27
- "tie_word_embeddings": false,
28
- "torch_dtype": "float32",
29
- "transformers_version": "4.41.0",
30
- "use_cache": true,
31
- "vocab_size": 102400
32
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/configuration_exaone.py DELETED
@@ -1,186 +0,0 @@
1
- # coding=utf-8
2
- # Copyright 2021 The LG AI Research EXAONE Lab. All rights reserved.
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
- """ EXAONE model configuration """
16
- from transformers.configuration_utils import PretrainedConfig
17
- from transformers.utils import logging
18
-
19
-
20
- logger = logging.get_logger(__name__)
21
-
22
- EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP = {
23
- }
24
-
25
-
26
- class ExaoneConfig(PretrainedConfig):
27
- r"""
28
- This is the configuration class to store the configuration of a :class:`~transformers.ExaoneModel`. It is used to
29
- instantiate a EXAONE model according to the specified arguments, defining the model architecture. Instantiating a
30
- configuration with the defaults will yield a similar configuration to that of the Exaone
31
-
32
- Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model
33
- outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information.
34
-
35
-
36
- Args:
37
- vocab_size (:obj:`int`, `optional`, defaults to 102400):
38
- Vocabulary size of the EXAONE model. Defines the number of different tokens that can be represented by the
39
- :obj:`inputs_ids` passed when calling :class:`~transformers.ExaoneModel`. Vocabulary size of the model.
40
- Defines the different tokens that can be represented by the `inputs_ids` passed to the forward method of
41
- :class:`~transformers.EXAONEModel`.
42
- max_position_embeddings (:obj:`int`, `optional`, defaults to 2048):
43
- The maximum sequence length that this model might ever be used with. Typically set this to something large
44
- just in case (e.g., 512 or 1024 or 2048).
45
- hidden_size (:obj:`int`, `optional`, defaults to 2048):
46
- Dimensionality of the encoder layers and the pooler layer.
47
- num_layers (:obj:`int`, `optional`, defaults to 32):
48
- Number of hidden layers in the Transformer encoder.
49
- num_attention_heads (:obj:`int`, `optional`, defaults to 32):
50
- Number of attention heads for each attention layer in the Transformer decoder.
51
- num_key_value_heads (:obj:`int`, `optional`):
52
- This is the number of key_value heads that should be used to implement Grouped Query Attention. If
53
- `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
54
- `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
55
- converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
56
- by meanpooling all the original heads within that group. For more details checkout [this
57
- paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
58
- `num_attention_heads`.
59
- intermediate_size (:obj:`int`, `optional`, defaults to `hidden_size * 4`):
60
- Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
61
- activation_function (:obj:`str` or :obj:`function`, `optional`, defaults to :obj:`"silu"`):
62
- The non-linear activation function (function or string) in the decoder.
63
- rope_theta (:obj:`float`, `optional`, defaults to 10000.0):
64
- The base period of the RoPE embeddings.
65
- rope_scaling (:obj:`Dict`, `optional`):
66
- Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
67
- and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
68
- accordingly.
69
- Expected contents:
70
- `rope_type` (:obj:`str`):
71
- The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
72
- 'llama3'], with 'default' being the original RoPE implementation.
73
- `factor` (:obj:`float`, `optional`):
74
- Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
75
- most scaling types, a `factor` of x will enable the model to handle sequences of length x *
76
- original maximum pre-trained length.
77
- `original_max_position_embeddings` (:obj:`int`, `optional`):
78
- Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
79
- pretraining.
80
- `attention_factor` (:obj:`float`, `optional`):
81
- Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
82
- computation. If unspecified, it defaults to value recommended by the implementation, using the
83
- `factor` field to infer the suggested value.
84
- `beta_fast` (:obj:`float`, `optional`):
85
- Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
86
- ramp function. If unspecified, it defaults to 32.
87
- `beta_slow` (:obj:`float`, `optional`):
88
- Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
89
- ramp function. If unspecified, it defaults to 1.
90
- `short_factor` (:obj:`List[float]`, `optional`):
91
- Only used with 'longrope'. The scaling factor to be applied to short contexts (<
92
- `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
93
- size divided by the number of attention heads divided by 2
94
- `long_factor` (:obj:`List[float]`, `optional`):
95
- Only used with 'longrope'. The scaling factor to be applied to long contexts (<
96
- `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
97
- size divided by the number of attention heads divided by 2
98
- `low_freq_factor` (:obj:`float`, `optional`):
99
- Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
100
- `high_freq_factor` (:obj:`float`, `optional`):
101
- Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
102
- embed_dropout (:obj:`float`, `optional`, defaults to 0.0):
103
- The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
104
- attention_dropout (:obj:`float`, `optional`, defaults to 0.0):
105
- The dropout ratio for the attention probabilities.
106
- layer_norm_epsilon (:obj:`float`, `optional`, defaults to 1e-5):
107
- The epsilon used by the layer normalization layers.
108
- initializer_range (:obj:`float`, `optional`, defaults to 0.02):
109
- The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
110
- use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`):
111
- Whether or not the model should return the last key/values attentions (not used by all models). Only
112
- relevant if ``config.is_decoder=True``.
113
- bos_token_id (:obj:`int`, `optional`, defaults to 0):
114
- Beginning of stream token id.
115
- eos_token_id (:obj:`int`, `optional`, defaults to 2):
116
- End of stream token id.
117
- tie_word_embeddings (:obj:`bool`, `optional`, defaults to :obj:`True`):
118
- Whether to tie weight embeddings
119
- gradient_checkpointing (:obj:`bool`, `optional`, defaults to :obj:`False`):
120
- If True, use gradient checkpointing to save memory at the expense of slower backward pass.
121
-
122
- Example::
123
-
124
- >>> from transformers import EXAONEModel, ExaoneConfig
125
-
126
- >>> # Initializing a EXAONE configuration
127
- >>> configuration = ExaoneConfig()
128
-
129
- >>> # Initializing a model from configuration
130
- >>> model = EXAONEModel(configuration)
131
-
132
- >>> # Accessing the model configuration
133
- >>> configuration = model.config
134
- """
135
- model_type = "exaone"
136
- keys_to_ignore_at_inference = ["past_key_values"]
137
- attribute_map = {"num_hidden_layers": "num_layers"}
138
-
139
- def __init__(
140
- self,
141
- vocab_size=102400,
142
- max_position_embeddings=2048,
143
- hidden_size=2048,
144
- num_layers=32,
145
- num_attention_heads=32,
146
- num_key_value_heads=None,
147
- intermediate_size=None,
148
- activation_function="silu",
149
- rope_theta=10000.0,
150
- rope_scaling=None,
151
- embed_dropout=0.0,
152
- attention_dropout=0.0,
153
- layer_norm_epsilon=1e-5,
154
- initializer_range=0.02,
155
- use_cache=True,
156
- bos_token_id=0,
157
- eos_token_id=2,
158
- tie_word_embeddings=True,
159
- **kwargs
160
- ):
161
- self.vocab_size = vocab_size
162
- self.max_position_embeddings = max_position_embeddings
163
- self.hidden_size = hidden_size
164
- self.num_layers = num_layers
165
- self.num_attention_heads = num_attention_heads
166
- self.num_hidden_layers = num_layers
167
- if num_key_value_heads is None:
168
- num_key_value_heads = num_attention_heads
169
- self.num_key_value_heads = num_key_value_heads
170
- if intermediate_size:
171
- self.intermediate_size = intermediate_size
172
- else:
173
- self.intermediate_size = hidden_size * 4
174
- self.activation_function = activation_function
175
- self.embed_dropout = embed_dropout
176
- self.attention_dropout = attention_dropout
177
- self.layer_norm_epsilon = layer_norm_epsilon
178
- self.initializer_range = initializer_range
179
- self.use_cache = use_cache
180
- self.rope_theta = rope_theta
181
- self.rope_scaling = rope_scaling
182
-
183
- self.bos_token_id = bos_token_id
184
- self.eos_token_id = eos_token_id
185
-
186
- super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/generation_config.json DELETED
@@ -1,7 +0,0 @@
1
- {
2
- "_from_model_config": true,
3
- "bos_token_id": 1,
4
- "eos_token_id": 361,
5
- "pad_token_id": 0,
6
- "transformers_version": "4.41"
7
- }
 
 
 
 
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/model-00001-of-00007.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c606ee52d0fa5bbec4fb9b1f0f00a26fae9ff7bc4494ebd1dfa69050f37d8eae
3
- size 4932636680
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/model-00002-of-00007.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7caeef4eee66dc420df77feb3352080e0295f4efb1295610b9a56024ed29d0da
3
- size 4999813040
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/model-00003-of-00007.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6dd2d4f8d6f580f331c24f7d6518acebe56957973b9cb0316c0b66fcaca5167a
3
- size 4999813080
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/model-00004-of-00007.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4f8a78c8e9926eaa9531b76681548d08811e721c460b439e46927ff6631f1916
3
- size 4832007464
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/model-00005-of-00007.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:03fb440bb35aa162943a7fd4603a68ece18f529374e784263d0b8352beb1b0e5
3
- size 4999813088
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/model-00006-of-00007.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3e17fb6e128f8b39c92742bf58459534cc2b27d4ad5dcc315f7be7aa572ade3a
3
- size 4832023944
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/model-00007-of-00007.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a9c431b6dce9f9f6283e28ef88d230aa1bedcb50f76cfa8146435b879e12fac0
3
- size 1677721728
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/model.safetensors.index.json DELETED
@@ -1,298 +0,0 @@
1
- {
2
- "metadata": {
3
- "total_size": 31273795584
4
- },
5
- "weight_map": {
6
- "lm_head.weight": "model-00007-of-00007.safetensors",
7
- "transformer.h.0.attn.attention.k_proj.weight": "model-00001-of-00007.safetensors",
8
- "transformer.h.0.attn.attention.out_proj.weight": "model-00001-of-00007.safetensors",
9
- "transformer.h.0.attn.attention.q_proj.weight": "model-00001-of-00007.safetensors",
10
- "transformer.h.0.attn.attention.v_proj.weight": "model-00001-of-00007.safetensors",
11
- "transformer.h.0.ln_1.weight": "model-00001-of-00007.safetensors",
12
- "transformer.h.0.ln_2.weight": "model-00001-of-00007.safetensors",
13
- "transformer.h.0.mlp.c_fc_0.weight": "model-00001-of-00007.safetensors",
14
- "transformer.h.0.mlp.c_fc_1.weight": "model-00001-of-00007.safetensors",
15
- "transformer.h.0.mlp.c_proj.weight": "model-00001-of-00007.safetensors",
16
- "transformer.h.1.attn.attention.k_proj.weight": "model-00001-of-00007.safetensors",
17
- "transformer.h.1.attn.attention.out_proj.weight": "model-00001-of-00007.safetensors",
18
- "transformer.h.1.attn.attention.q_proj.weight": "model-00001-of-00007.safetensors",
19
- "transformer.h.1.attn.attention.v_proj.weight": "model-00001-of-00007.safetensors",
20
- "transformer.h.1.ln_1.weight": "model-00001-of-00007.safetensors",
21
- "transformer.h.1.ln_2.weight": "model-00001-of-00007.safetensors",
22
- "transformer.h.1.mlp.c_fc_0.weight": "model-00001-of-00007.safetensors",
23
- "transformer.h.1.mlp.c_fc_1.weight": "model-00001-of-00007.safetensors",
24
- "transformer.h.1.mlp.c_proj.weight": "model-00001-of-00007.safetensors",
25
- "transformer.h.10.attn.attention.k_proj.weight": "model-00003-of-00007.safetensors",
26
- "transformer.h.10.attn.attention.out_proj.weight": "model-00003-of-00007.safetensors",
27
- "transformer.h.10.attn.attention.q_proj.weight": "model-00003-of-00007.safetensors",
28
- "transformer.h.10.attn.attention.v_proj.weight": "model-00003-of-00007.safetensors",
29
- "transformer.h.10.ln_1.weight": "model-00003-of-00007.safetensors",
30
- "transformer.h.10.ln_2.weight": "model-00003-of-00007.safetensors",
31
- "transformer.h.10.mlp.c_fc_0.weight": "model-00003-of-00007.safetensors",
32
- "transformer.h.10.mlp.c_fc_1.weight": "model-00003-of-00007.safetensors",
33
- "transformer.h.10.mlp.c_proj.weight": "model-00003-of-00007.safetensors",
34
- "transformer.h.11.attn.attention.k_proj.weight": "model-00003-of-00007.safetensors",
35
- "transformer.h.11.attn.attention.out_proj.weight": "model-00003-of-00007.safetensors",
36
- "transformer.h.11.attn.attention.q_proj.weight": "model-00003-of-00007.safetensors",
37
- "transformer.h.11.attn.attention.v_proj.weight": "model-00003-of-00007.safetensors",
38
- "transformer.h.11.ln_1.weight": "model-00003-of-00007.safetensors",
39
- "transformer.h.11.ln_2.weight": "model-00003-of-00007.safetensors",
40
- "transformer.h.11.mlp.c_fc_0.weight": "model-00003-of-00007.safetensors",
41
- "transformer.h.11.mlp.c_fc_1.weight": "model-00003-of-00007.safetensors",
42
- "transformer.h.11.mlp.c_proj.weight": "model-00003-of-00007.safetensors",
43
- "transformer.h.12.attn.attention.k_proj.weight": "model-00003-of-00007.safetensors",
44
- "transformer.h.12.attn.attention.out_proj.weight": "model-00003-of-00007.safetensors",
45
- "transformer.h.12.attn.attention.q_proj.weight": "model-00003-of-00007.safetensors",
46
- "transformer.h.12.attn.attention.v_proj.weight": "model-00003-of-00007.safetensors",
47
- "transformer.h.12.ln_1.weight": "model-00003-of-00007.safetensors",
48
- "transformer.h.12.ln_2.weight": "model-00003-of-00007.safetensors",
49
- "transformer.h.12.mlp.c_fc_0.weight": "model-00003-of-00007.safetensors",
50
- "transformer.h.12.mlp.c_fc_1.weight": "model-00003-of-00007.safetensors",
51
- "transformer.h.12.mlp.c_proj.weight": "model-00003-of-00007.safetensors",
52
- "transformer.h.13.attn.attention.k_proj.weight": "model-00003-of-00007.safetensors",
53
- "transformer.h.13.attn.attention.out_proj.weight": "model-00003-of-00007.safetensors",
54
- "transformer.h.13.attn.attention.q_proj.weight": "model-00003-of-00007.safetensors",
55
- "transformer.h.13.attn.attention.v_proj.weight": "model-00003-of-00007.safetensors",
56
- "transformer.h.13.ln_1.weight": "model-00003-of-00007.safetensors",
57
- "transformer.h.13.ln_2.weight": "model-00003-of-00007.safetensors",
58
- "transformer.h.13.mlp.c_fc_0.weight": "model-00003-of-00007.safetensors",
59
- "transformer.h.13.mlp.c_fc_1.weight": "model-00003-of-00007.safetensors",
60
- "transformer.h.13.mlp.c_proj.weight": "model-00003-of-00007.safetensors",
61
- "transformer.h.14.attn.attention.k_proj.weight": "model-00003-of-00007.safetensors",
62
- "transformer.h.14.attn.attention.out_proj.weight": "model-00003-of-00007.safetensors",
63
- "transformer.h.14.attn.attention.q_proj.weight": "model-00003-of-00007.safetensors",
64
- "transformer.h.14.attn.attention.v_proj.weight": "model-00003-of-00007.safetensors",
65
- "transformer.h.14.ln_1.weight": "model-00003-of-00007.safetensors",
66
- "transformer.h.14.ln_2.weight": "model-00003-of-00007.safetensors",
67
- "transformer.h.14.mlp.c_fc_0.weight": "model-00003-of-00007.safetensors",
68
- "transformer.h.14.mlp.c_fc_1.weight": "model-00003-of-00007.safetensors",
69
- "transformer.h.14.mlp.c_proj.weight": "model-00003-of-00007.safetensors",
70
- "transformer.h.15.attn.attention.k_proj.weight": "model-00003-of-00007.safetensors",
71
- "transformer.h.15.attn.attention.out_proj.weight": "model-00003-of-00007.safetensors",
72
- "transformer.h.15.attn.attention.q_proj.weight": "model-00003-of-00007.safetensors",
73
- "transformer.h.15.attn.attention.v_proj.weight": "model-00003-of-00007.safetensors",
74
- "transformer.h.15.ln_1.weight": "model-00003-of-00007.safetensors",
75
- "transformer.h.15.ln_2.weight": "model-00003-of-00007.safetensors",
76
- "transformer.h.15.mlp.c_fc_0.weight": "model-00004-of-00007.safetensors",
77
- "transformer.h.15.mlp.c_fc_1.weight": "model-00004-of-00007.safetensors",
78
- "transformer.h.15.mlp.c_proj.weight": "model-00004-of-00007.safetensors",
79
- "transformer.h.16.attn.attention.k_proj.weight": "model-00004-of-00007.safetensors",
80
- "transformer.h.16.attn.attention.out_proj.weight": "model-00004-of-00007.safetensors",
81
- "transformer.h.16.attn.attention.q_proj.weight": "model-00004-of-00007.safetensors",
82
- "transformer.h.16.attn.attention.v_proj.weight": "model-00004-of-00007.safetensors",
83
- "transformer.h.16.ln_1.weight": "model-00004-of-00007.safetensors",
84
- "transformer.h.16.ln_2.weight": "model-00004-of-00007.safetensors",
85
- "transformer.h.16.mlp.c_fc_0.weight": "model-00004-of-00007.safetensors",
86
- "transformer.h.16.mlp.c_fc_1.weight": "model-00004-of-00007.safetensors",
87
- "transformer.h.16.mlp.c_proj.weight": "model-00004-of-00007.safetensors",
88
- "transformer.h.17.attn.attention.k_proj.weight": "model-00004-of-00007.safetensors",
89
- "transformer.h.17.attn.attention.out_proj.weight": "model-00004-of-00007.safetensors",
90
- "transformer.h.17.attn.attention.q_proj.weight": "model-00004-of-00007.safetensors",
91
- "transformer.h.17.attn.attention.v_proj.weight": "model-00004-of-00007.safetensors",
92
- "transformer.h.17.ln_1.weight": "model-00004-of-00007.safetensors",
93
- "transformer.h.17.ln_2.weight": "model-00004-of-00007.safetensors",
94
- "transformer.h.17.mlp.c_fc_0.weight": "model-00004-of-00007.safetensors",
95
- "transformer.h.17.mlp.c_fc_1.weight": "model-00004-of-00007.safetensors",
96
- "transformer.h.17.mlp.c_proj.weight": "model-00004-of-00007.safetensors",
97
- "transformer.h.18.attn.attention.k_proj.weight": "model-00004-of-00007.safetensors",
98
- "transformer.h.18.attn.attention.out_proj.weight": "model-00004-of-00007.safetensors",
99
- "transformer.h.18.attn.attention.q_proj.weight": "model-00004-of-00007.safetensors",
100
- "transformer.h.18.attn.attention.v_proj.weight": "model-00004-of-00007.safetensors",
101
- "transformer.h.18.ln_1.weight": "model-00004-of-00007.safetensors",
102
- "transformer.h.18.ln_2.weight": "model-00004-of-00007.safetensors",
103
- "transformer.h.18.mlp.c_fc_0.weight": "model-00004-of-00007.safetensors",
104
- "transformer.h.18.mlp.c_fc_1.weight": "model-00004-of-00007.safetensors",
105
- "transformer.h.18.mlp.c_proj.weight": "model-00004-of-00007.safetensors",
106
- "transformer.h.19.attn.attention.k_proj.weight": "model-00004-of-00007.safetensors",
107
- "transformer.h.19.attn.attention.out_proj.weight": "model-00004-of-00007.safetensors",
108
- "transformer.h.19.attn.attention.q_proj.weight": "model-00004-of-00007.safetensors",
109
- "transformer.h.19.attn.attention.v_proj.weight": "model-00004-of-00007.safetensors",
110
- "transformer.h.19.ln_1.weight": "model-00004-of-00007.safetensors",
111
- "transformer.h.19.ln_2.weight": "model-00004-of-00007.safetensors",
112
- "transformer.h.19.mlp.c_fc_0.weight": "model-00004-of-00007.safetensors",
113
- "transformer.h.19.mlp.c_fc_1.weight": "model-00004-of-00007.safetensors",
114
- "transformer.h.19.mlp.c_proj.weight": "model-00004-of-00007.safetensors",
115
- "transformer.h.2.attn.attention.k_proj.weight": "model-00001-of-00007.safetensors",
116
- "transformer.h.2.attn.attention.out_proj.weight": "model-00001-of-00007.safetensors",
117
- "transformer.h.2.attn.attention.q_proj.weight": "model-00001-of-00007.safetensors",
118
- "transformer.h.2.attn.attention.v_proj.weight": "model-00001-of-00007.safetensors",
119
- "transformer.h.2.ln_1.weight": "model-00001-of-00007.safetensors",
120
- "transformer.h.2.ln_2.weight": "model-00001-of-00007.safetensors",
121
- "transformer.h.2.mlp.c_fc_0.weight": "model-00001-of-00007.safetensors",
122
- "transformer.h.2.mlp.c_fc_1.weight": "model-00001-of-00007.safetensors",
123
- "transformer.h.2.mlp.c_proj.weight": "model-00001-of-00007.safetensors",
124
- "transformer.h.20.attn.attention.k_proj.weight": "model-00004-of-00007.safetensors",
125
- "transformer.h.20.attn.attention.out_proj.weight": "model-00004-of-00007.safetensors",
126
- "transformer.h.20.attn.attention.q_proj.weight": "model-00004-of-00007.safetensors",
127
- "transformer.h.20.attn.attention.v_proj.weight": "model-00004-of-00007.safetensors",
128
- "transformer.h.20.ln_1.weight": "model-00004-of-00007.safetensors",
129
- "transformer.h.20.ln_2.weight": "model-00004-of-00007.safetensors",
130
- "transformer.h.20.mlp.c_fc_0.weight": "model-00004-of-00007.safetensors",
131
- "transformer.h.20.mlp.c_fc_1.weight": "model-00004-of-00007.safetensors",
132
- "transformer.h.20.mlp.c_proj.weight": "model-00005-of-00007.safetensors",
133
- "transformer.h.21.attn.attention.k_proj.weight": "model-00005-of-00007.safetensors",
134
- "transformer.h.21.attn.attention.out_proj.weight": "model-00005-of-00007.safetensors",
135
- "transformer.h.21.attn.attention.q_proj.weight": "model-00005-of-00007.safetensors",
136
- "transformer.h.21.attn.attention.v_proj.weight": "model-00005-of-00007.safetensors",
137
- "transformer.h.21.ln_1.weight": "model-00005-of-00007.safetensors",
138
- "transformer.h.21.ln_2.weight": "model-00005-of-00007.safetensors",
139
- "transformer.h.21.mlp.c_fc_0.weight": "model-00005-of-00007.safetensors",
140
- "transformer.h.21.mlp.c_fc_1.weight": "model-00005-of-00007.safetensors",
141
- "transformer.h.21.mlp.c_proj.weight": "model-00005-of-00007.safetensors",
142
- "transformer.h.22.attn.attention.k_proj.weight": "model-00005-of-00007.safetensors",
143
- "transformer.h.22.attn.attention.out_proj.weight": "model-00005-of-00007.safetensors",
144
- "transformer.h.22.attn.attention.q_proj.weight": "model-00005-of-00007.safetensors",
145
- "transformer.h.22.attn.attention.v_proj.weight": "model-00005-of-00007.safetensors",
146
- "transformer.h.22.ln_1.weight": "model-00005-of-00007.safetensors",
147
- "transformer.h.22.ln_2.weight": "model-00005-of-00007.safetensors",
148
- "transformer.h.22.mlp.c_fc_0.weight": "model-00005-of-00007.safetensors",
149
- "transformer.h.22.mlp.c_fc_1.weight": "model-00005-of-00007.safetensors",
150
- "transformer.h.22.mlp.c_proj.weight": "model-00005-of-00007.safetensors",
151
- "transformer.h.23.attn.attention.k_proj.weight": "model-00005-of-00007.safetensors",
152
- "transformer.h.23.attn.attention.out_proj.weight": "model-00005-of-00007.safetensors",
153
- "transformer.h.23.attn.attention.q_proj.weight": "model-00005-of-00007.safetensors",
154
- "transformer.h.23.attn.attention.v_proj.weight": "model-00005-of-00007.safetensors",
155
- "transformer.h.23.ln_1.weight": "model-00005-of-00007.safetensors",
156
- "transformer.h.23.ln_2.weight": "model-00005-of-00007.safetensors",
157
- "transformer.h.23.mlp.c_fc_0.weight": "model-00005-of-00007.safetensors",
158
- "transformer.h.23.mlp.c_fc_1.weight": "model-00005-of-00007.safetensors",
159
- "transformer.h.23.mlp.c_proj.weight": "model-00005-of-00007.safetensors",
160
- "transformer.h.24.attn.attention.k_proj.weight": "model-00005-of-00007.safetensors",
161
- "transformer.h.24.attn.attention.out_proj.weight": "model-00005-of-00007.safetensors",
162
- "transformer.h.24.attn.attention.q_proj.weight": "model-00005-of-00007.safetensors",
163
- "transformer.h.24.attn.attention.v_proj.weight": "model-00005-of-00007.safetensors",
164
- "transformer.h.24.ln_1.weight": "model-00005-of-00007.safetensors",
165
- "transformer.h.24.ln_2.weight": "model-00005-of-00007.safetensors",
166
- "transformer.h.24.mlp.c_fc_0.weight": "model-00005-of-00007.safetensors",
167
- "transformer.h.24.mlp.c_fc_1.weight": "model-00005-of-00007.safetensors",
168
- "transformer.h.24.mlp.c_proj.weight": "model-00005-of-00007.safetensors",
169
- "transformer.h.25.attn.attention.k_proj.weight": "model-00005-of-00007.safetensors",
170
- "transformer.h.25.attn.attention.out_proj.weight": "model-00005-of-00007.safetensors",
171
- "transformer.h.25.attn.attention.q_proj.weight": "model-00005-of-00007.safetensors",
172
- "transformer.h.25.attn.attention.v_proj.weight": "model-00005-of-00007.safetensors",
173
- "transformer.h.25.ln_1.weight": "model-00005-of-00007.safetensors",
174
- "transformer.h.25.ln_2.weight": "model-00005-of-00007.safetensors",
175
- "transformer.h.25.mlp.c_fc_0.weight": "model-00005-of-00007.safetensors",
176
- "transformer.h.25.mlp.c_fc_1.weight": "model-00005-of-00007.safetensors",
177
- "transformer.h.25.mlp.c_proj.weight": "model-00005-of-00007.safetensors",
178
- "transformer.h.26.attn.attention.k_proj.weight": "model-00005-of-00007.safetensors",
179
- "transformer.h.26.attn.attention.out_proj.weight": "model-00005-of-00007.safetensors",
180
- "transformer.h.26.attn.attention.q_proj.weight": "model-00005-of-00007.safetensors",
181
- "transformer.h.26.attn.attention.v_proj.weight": "model-00005-of-00007.safetensors",
182
- "transformer.h.26.ln_1.weight": "model-00005-of-00007.safetensors",
183
- "transformer.h.26.ln_2.weight": "model-00005-of-00007.safetensors",
184
- "transformer.h.26.mlp.c_fc_0.weight": "model-00005-of-00007.safetensors",
185
- "transformer.h.26.mlp.c_fc_1.weight": "model-00006-of-00007.safetensors",
186
- "transformer.h.26.mlp.c_proj.weight": "model-00006-of-00007.safetensors",
187
- "transformer.h.27.attn.attention.k_proj.weight": "model-00006-of-00007.safetensors",
188
- "transformer.h.27.attn.attention.out_proj.weight": "model-00006-of-00007.safetensors",
189
- "transformer.h.27.attn.attention.q_proj.weight": "model-00006-of-00007.safetensors",
190
- "transformer.h.27.attn.attention.v_proj.weight": "model-00006-of-00007.safetensors",
191
- "transformer.h.27.ln_1.weight": "model-00006-of-00007.safetensors",
192
- "transformer.h.27.ln_2.weight": "model-00006-of-00007.safetensors",
193
- "transformer.h.27.mlp.c_fc_0.weight": "model-00006-of-00007.safetensors",
194
- "transformer.h.27.mlp.c_fc_1.weight": "model-00006-of-00007.safetensors",
195
- "transformer.h.27.mlp.c_proj.weight": "model-00006-of-00007.safetensors",
196
- "transformer.h.28.attn.attention.k_proj.weight": "model-00006-of-00007.safetensors",
197
- "transformer.h.28.attn.attention.out_proj.weight": "model-00006-of-00007.safetensors",
198
- "transformer.h.28.attn.attention.q_proj.weight": "model-00006-of-00007.safetensors",
199
- "transformer.h.28.attn.attention.v_proj.weight": "model-00006-of-00007.safetensors",
200
- "transformer.h.28.ln_1.weight": "model-00006-of-00007.safetensors",
201
- "transformer.h.28.ln_2.weight": "model-00006-of-00007.safetensors",
202
- "transformer.h.28.mlp.c_fc_0.weight": "model-00006-of-00007.safetensors",
203
- "transformer.h.28.mlp.c_fc_1.weight": "model-00006-of-00007.safetensors",
204
- "transformer.h.28.mlp.c_proj.weight": "model-00006-of-00007.safetensors",
205
- "transformer.h.29.attn.attention.k_proj.weight": "model-00006-of-00007.safetensors",
206
- "transformer.h.29.attn.attention.out_proj.weight": "model-00006-of-00007.safetensors",
207
- "transformer.h.29.attn.attention.q_proj.weight": "model-00006-of-00007.safetensors",
208
- "transformer.h.29.attn.attention.v_proj.weight": "model-00006-of-00007.safetensors",
209
- "transformer.h.29.ln_1.weight": "model-00006-of-00007.safetensors",
210
- "transformer.h.29.ln_2.weight": "model-00006-of-00007.safetensors",
211
- "transformer.h.29.mlp.c_fc_0.weight": "model-00006-of-00007.safetensors",
212
- "transformer.h.29.mlp.c_fc_1.weight": "model-00006-of-00007.safetensors",
213
- "transformer.h.29.mlp.c_proj.weight": "model-00006-of-00007.safetensors",
214
- "transformer.h.3.attn.attention.k_proj.weight": "model-00001-of-00007.safetensors",
215
- "transformer.h.3.attn.attention.out_proj.weight": "model-00001-of-00007.safetensors",
216
- "transformer.h.3.attn.attention.q_proj.weight": "model-00001-of-00007.safetensors",
217
- "transformer.h.3.attn.attention.v_proj.weight": "model-00001-of-00007.safetensors",
218
- "transformer.h.3.ln_1.weight": "model-00001-of-00007.safetensors",
219
- "transformer.h.3.ln_2.weight": "model-00001-of-00007.safetensors",
220
- "transformer.h.3.mlp.c_fc_0.weight": "model-00001-of-00007.safetensors",
221
- "transformer.h.3.mlp.c_fc_1.weight": "model-00001-of-00007.safetensors",
222
- "transformer.h.3.mlp.c_proj.weight": "model-00002-of-00007.safetensors",
223
- "transformer.h.30.attn.attention.k_proj.weight": "model-00006-of-00007.safetensors",
224
- "transformer.h.30.attn.attention.out_proj.weight": "model-00006-of-00007.safetensors",
225
- "transformer.h.30.attn.attention.q_proj.weight": "model-00006-of-00007.safetensors",
226
- "transformer.h.30.attn.attention.v_proj.weight": "model-00006-of-00007.safetensors",
227
- "transformer.h.30.ln_1.weight": "model-00006-of-00007.safetensors",
228
- "transformer.h.30.ln_2.weight": "model-00006-of-00007.safetensors",
229
- "transformer.h.30.mlp.c_fc_0.weight": "model-00006-of-00007.safetensors",
230
- "transformer.h.30.mlp.c_fc_1.weight": "model-00006-of-00007.safetensors",
231
- "transformer.h.30.mlp.c_proj.weight": "model-00006-of-00007.safetensors",
232
- "transformer.h.31.attn.attention.k_proj.weight": "model-00006-of-00007.safetensors",
233
- "transformer.h.31.attn.attention.out_proj.weight": "model-00006-of-00007.safetensors",
234
- "transformer.h.31.attn.attention.q_proj.weight": "model-00006-of-00007.safetensors",
235
- "transformer.h.31.attn.attention.v_proj.weight": "model-00006-of-00007.safetensors",
236
- "transformer.h.31.ln_1.weight": "model-00006-of-00007.safetensors",
237
- "transformer.h.31.ln_2.weight": "model-00006-of-00007.safetensors",
238
- "transformer.h.31.mlp.c_fc_0.weight": "model-00006-of-00007.safetensors",
239
- "transformer.h.31.mlp.c_fc_1.weight": "model-00006-of-00007.safetensors",
240
- "transformer.h.31.mlp.c_proj.weight": "model-00006-of-00007.safetensors",
241
- "transformer.h.4.attn.attention.k_proj.weight": "model-00002-of-00007.safetensors",
242
- "transformer.h.4.attn.attention.out_proj.weight": "model-00002-of-00007.safetensors",
243
- "transformer.h.4.attn.attention.q_proj.weight": "model-00002-of-00007.safetensors",
244
- "transformer.h.4.attn.attention.v_proj.weight": "model-00002-of-00007.safetensors",
245
- "transformer.h.4.ln_1.weight": "model-00002-of-00007.safetensors",
246
- "transformer.h.4.ln_2.weight": "model-00002-of-00007.safetensors",
247
- "transformer.h.4.mlp.c_fc_0.weight": "model-00002-of-00007.safetensors",
248
- "transformer.h.4.mlp.c_fc_1.weight": "model-00002-of-00007.safetensors",
249
- "transformer.h.4.mlp.c_proj.weight": "model-00002-of-00007.safetensors",
250
- "transformer.h.5.attn.attention.k_proj.weight": "model-00002-of-00007.safetensors",
251
- "transformer.h.5.attn.attention.out_proj.weight": "model-00002-of-00007.safetensors",
252
- "transformer.h.5.attn.attention.q_proj.weight": "model-00002-of-00007.safetensors",
253
- "transformer.h.5.attn.attention.v_proj.weight": "model-00002-of-00007.safetensors",
254
- "transformer.h.5.ln_1.weight": "model-00002-of-00007.safetensors",
255
- "transformer.h.5.ln_2.weight": "model-00002-of-00007.safetensors",
256
- "transformer.h.5.mlp.c_fc_0.weight": "model-00002-of-00007.safetensors",
257
- "transformer.h.5.mlp.c_fc_1.weight": "model-00002-of-00007.safetensors",
258
- "transformer.h.5.mlp.c_proj.weight": "model-00002-of-00007.safetensors",
259
- "transformer.h.6.attn.attention.k_proj.weight": "model-00002-of-00007.safetensors",
260
- "transformer.h.6.attn.attention.out_proj.weight": "model-00002-of-00007.safetensors",
261
- "transformer.h.6.attn.attention.q_proj.weight": "model-00002-of-00007.safetensors",
262
- "transformer.h.6.attn.attention.v_proj.weight": "model-00002-of-00007.safetensors",
263
- "transformer.h.6.ln_1.weight": "model-00002-of-00007.safetensors",
264
- "transformer.h.6.ln_2.weight": "model-00002-of-00007.safetensors",
265
- "transformer.h.6.mlp.c_fc_0.weight": "model-00002-of-00007.safetensors",
266
- "transformer.h.6.mlp.c_fc_1.weight": "model-00002-of-00007.safetensors",
267
- "transformer.h.6.mlp.c_proj.weight": "model-00002-of-00007.safetensors",
268
- "transformer.h.7.attn.attention.k_proj.weight": "model-00002-of-00007.safetensors",
269
- "transformer.h.7.attn.attention.out_proj.weight": "model-00002-of-00007.safetensors",
270
- "transformer.h.7.attn.attention.q_proj.weight": "model-00002-of-00007.safetensors",
271
- "transformer.h.7.attn.attention.v_proj.weight": "model-00002-of-00007.safetensors",
272
- "transformer.h.7.ln_1.weight": "model-00002-of-00007.safetensors",
273
- "transformer.h.7.ln_2.weight": "model-00002-of-00007.safetensors",
274
- "transformer.h.7.mlp.c_fc_0.weight": "model-00002-of-00007.safetensors",
275
- "transformer.h.7.mlp.c_fc_1.weight": "model-00002-of-00007.safetensors",
276
- "transformer.h.7.mlp.c_proj.weight": "model-00002-of-00007.safetensors",
277
- "transformer.h.8.attn.attention.k_proj.weight": "model-00002-of-00007.safetensors",
278
- "transformer.h.8.attn.attention.out_proj.weight": "model-00002-of-00007.safetensors",
279
- "transformer.h.8.attn.attention.q_proj.weight": "model-00002-of-00007.safetensors",
280
- "transformer.h.8.attn.attention.v_proj.weight": "model-00002-of-00007.safetensors",
281
- "transformer.h.8.ln_1.weight": "model-00002-of-00007.safetensors",
282
- "transformer.h.8.ln_2.weight": "model-00002-of-00007.safetensors",
283
- "transformer.h.8.mlp.c_fc_0.weight": "model-00002-of-00007.safetensors",
284
- "transformer.h.8.mlp.c_fc_1.weight": "model-00002-of-00007.safetensors",
285
- "transformer.h.8.mlp.c_proj.weight": "model-00002-of-00007.safetensors",
286
- "transformer.h.9.attn.attention.k_proj.weight": "model-00002-of-00007.safetensors",
287
- "transformer.h.9.attn.attention.out_proj.weight": "model-00002-of-00007.safetensors",
288
- "transformer.h.9.attn.attention.q_proj.weight": "model-00002-of-00007.safetensors",
289
- "transformer.h.9.attn.attention.v_proj.weight": "model-00002-of-00007.safetensors",
290
- "transformer.h.9.ln_1.weight": "model-00002-of-00007.safetensors",
291
- "transformer.h.9.ln_2.weight": "model-00002-of-00007.safetensors",
292
- "transformer.h.9.mlp.c_fc_0.weight": "model-00002-of-00007.safetensors",
293
- "transformer.h.9.mlp.c_fc_1.weight": "model-00003-of-00007.safetensors",
294
- "transformer.h.9.mlp.c_proj.weight": "model-00003-of-00007.safetensors",
295
- "transformer.ln_f.weight": "model-00006-of-00007.safetensors",
296
- "transformer.wte.weight": "model-00001-of-00007.safetensors"
297
- }
298
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7f15baedd46858153d817445aff032f4d6cf4939/modeling_exaone.py DELETED
@@ -1,1747 +0,0 @@
1
- # coding=utf-8
2
- # Copyright 2021 The LG AI Research EXAONE Lab
3
- # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
4
- #
5
- # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
6
- # and OPT implementations in this library. It has been modified from its
7
- # original forms to accommodate minor architectural differences compared
8
- # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
9
- #
10
- # Licensed under the Apache License, Version 2.0 (the "License");
11
- # you may not use this file except in compliance with the License.
12
- # You may obtain a copy of the License at
13
- #
14
- # http://www.apache.org/licenses/LICENSE-2.0
15
- #
16
- # Unless required by applicable law or agreed to in writing, software
17
- # distributed under the License is distributed on an "AS IS" BASIS,
18
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
- # See the License for the specific language governing permissions and
20
- # limitations under the License.
21
- """ LG AI Research EXAONE Lab"""
22
- import sys
23
- import os
24
- from typing import List, Optional, Tuple, Union
25
- from packaging import version
26
-
27
- import torch
28
- import torch.utils.checkpoint
29
- from torch import nn
30
- from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
31
- import torch.nn.functional as F
32
-
33
- from transformers.activations import ACT2FN
34
- from transformers.cache_utils import Cache, DynamicCache, StaticCache
35
- from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
36
- from transformers.configuration_utils import PretrainedConfig
37
- from transformers.modeling_attn_mask_utils import AttentionMaskConverter
38
-
39
- from transformers.modeling_outputs import (
40
- BaseModelOutputWithPast,
41
- BaseModelOutputWithPastAndCrossAttentions,
42
- CausalLMOutputWithCrossAttentions,
43
- CausalLMOutputWithPast,
44
- SequenceClassifierOutputWithPast,
45
- QuestionAnsweringModelOutput,
46
- )
47
- from transformers.modeling_utils import PreTrainedModel
48
- from transformers.utils import (
49
- add_code_sample_docstrings,
50
- add_start_docstrings,
51
- add_start_docstrings_to_model_forward,
52
- is_flash_attn_2_available,
53
- logging,
54
- )
55
- from .configuration_exaone import ExaoneConfig
56
- from torch.nn.utils import skip_init
57
- import math
58
- import numpy as np
59
- from typing import List, Optional, Tuple, Union
60
-
61
-
62
- if is_flash_attn_2_available():
63
- try:
64
- import inspect
65
- from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input
66
- from flash_attn import flash_attn_func, flash_attn_varlen_func
67
-
68
- _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters)
69
-
70
- import flash_attn
71
- if version.parse(flash_attn.__version__) > version.parse('2.4.2'):
72
- from flash_attn.ops.triton.layer_norm import rms_norm_fn
73
- else:
74
- from flash_attn.ops.triton.layernorm import rms_norm_fn
75
- except:
76
- pass
77
-
78
-
79
- logger = logging.get_logger(__name__)
80
-
81
- _CHECKPOINT_FOR_DOC = "exaone"
82
- _CONFIG_FOR_DOC = "ExaoneConfig"
83
-
84
- EXAONE_PRETRAINED_MODEL_ARCHIVE_LIST = [
85
- "exaone",
86
- ]
87
-
88
-
89
- @torch.jit.script
90
- def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
91
- """
92
- This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
93
- num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
94
- """
95
- batch, num_key_value_heads, slen, head_dim = hidden_states.shape
96
- if n_rep == 1:
97
- return hidden_states
98
- hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
99
- return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
100
-
101
-
102
- def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
103
- """Applies Rotary Position Embedding to the query and key tensors.
104
-
105
- Args:
106
- q (`torch.Tensor`): The query tensor.
107
- k (`torch.Tensor`): The key tensor.
108
- cos (`torch.Tensor`): The cosine part of the rotary embedding.
109
- sin (`torch.Tensor`): The sine part of the rotary embedding.
110
- unsqueeze_dim (`int`, *optional*, defaults to 1):
111
- The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
112
- sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
113
- that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
114
- k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
115
- cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
116
- the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
117
- Returns:
118
- `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
119
- """
120
- cos = cos.unsqueeze(unsqueeze_dim)
121
- sin = sin.unsqueeze(unsqueeze_dim)
122
- q_embed = (q * cos) + (rotate_half(q) * sin)
123
- k_embed = (k * cos) + (rotate_half(k) * sin)
124
- return q_embed, k_embed
125
-
126
-
127
- def rotate_half(x):
128
- """ Rotates half the hidden dims of the input. """
129
- x1 = x[..., : x.shape[-1] // 2]
130
- x2 = x[..., x.shape[-1] // 2 :]
131
- return torch.cat((-x2, x1), dim=-1)
132
-
133
-
134
- # copied from llama
135
- def _prepare_4d_causal_attention_mask_with_cache_position(
136
- attention_mask: torch.Tensor,
137
- sequence_length: int,
138
- target_length: int,
139
- dtype: torch.dtype,
140
- device: torch.device,
141
- min_dtype: float,
142
- cache_position: torch.Tensor,
143
- batch_size: int,
144
- ):
145
- """
146
- Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
147
- `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
148
-
149
- Args:
150
- attention_mask (`torch.Tensor`):
151
- A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
152
- sequence_length (`int`):
153
- The sequence length being processed.
154
- target_length (`int`):
155
- The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
156
- dtype (`torch.dtype`):
157
- The dtype to use for the 4D attention mask.
158
- device (`torch.device`):
159
- The device to plcae the 4D attention mask on.
160
- min_dtype (`float`):
161
- The minimum value representable with the dtype `dtype`.
162
- cache_position (`torch.Tensor`):
163
- Indices depicting the position of the input sequence tokens in the sequence.
164
- batch_size (`torch.Tensor`):
165
- Batch size.
166
- """
167
- if attention_mask is not None and attention_mask.dim() == 4:
168
- # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
169
- causal_mask = attention_mask
170
- else:
171
- causal_mask = torch.full((sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device)
172
- if sequence_length != 1:
173
- causal_mask = torch.triu(causal_mask, diagonal=1)
174
- causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1)
175
- causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
176
- if attention_mask is not None:
177
- causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit
178
- mask_length = attention_mask.shape[-1]
179
- padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :]
180
- padding_mask = padding_mask == 0
181
- causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(
182
- padding_mask, min_dtype
183
- )
184
-
185
- return causal_mask
186
-
187
-
188
- class ExaoneRMSNorm(torch.nn.Module):
189
- def __init__(self, hidden_size, eps=1e-6):
190
- super().__init__()
191
- self.eps = eps
192
- self.weight = torch.nn.Parameter(torch.ones(hidden_size))
193
-
194
- def forward(self, hidden_states):
195
- input_dtype = hidden_states.dtype
196
- hidden_states = hidden_states.to(torch.float32)
197
- variance = hidden_states.pow(2).mean(-1, keepdim=True)
198
- hidden_states = hidden_states * torch.rsqrt(variance + self.eps)
199
- return self.weight * hidden_states.to(input_dtype)
200
-
201
-
202
- class ExaoneTritonRMSNorm(torch.nn.Module):
203
- def __init__(
204
- self,
205
- hidden_size: int = 0,
206
- eps: float = 1e-5,
207
- ):
208
- super().__init__()
209
- self.eps = eps
210
- self.drop = None
211
- self.weight = torch.nn.Parameter(torch.empty(hidden_size))
212
- self.register_parameter("bias", None)
213
- self.reset_parameters()
214
-
215
- def reset_parameters(self):
216
- torch.nn.init.ones_(self.weight)
217
-
218
- def forward(self, x, residual=None, prenorm=False, residual_in_fp32=False):
219
- return rms_norm_fn(
220
- x,
221
- self.weight,
222
- self.bias,
223
- residual=residual,
224
- eps=self.eps,
225
- dropout_p=self.drop.p if self.drop is not None and self.training else 0.0,
226
- prenorm=prenorm,
227
- residual_in_fp32=residual_in_fp32,
228
- )
229
-
230
-
231
- ALL_LAYERNORM_LAYERS.append(ExaoneRMSNorm)
232
- ALL_LAYERNORM_LAYERS.append(ExaoneTritonRMSNorm)
233
-
234
-
235
- class ExaoneRotaryEmbedding(nn.Module):
236
- """
237
- Common description for the functions named `_compute_XXX_rope_parameters()`
238
- - Copied from `transformers.modeling_rope_utils` in v4.43, with some modifications.
239
-
240
- Computes the inverse frequencies with linear scaling.
241
- The EXAONE model supports 'default', 'linear', 'dynamic', and 'yarn'.
242
-
243
- Args:
244
- config (:obj:`~transformers.PretrainedConfig`):
245
- The model configuration.
246
- device (:obj:`torch.device`):
247
- The device to use for initialization of the inverse frequencies.
248
- seq_len (:obj:`int`, `optional`):
249
- The current sequence length. Unused for this type of RoPE.
250
- Returns:
251
- Tuple of (:obj:`torch.Tensor`, :obj:`float`), containing the inverse frequencies for the RoPE embeddings and the
252
- post-processing scaling factor applied to the computed cos/sin (unused in some types of RoPE).
253
- """
254
-
255
- def _compute_default_rope_parameters(
256
- self,
257
- config: Optional[PretrainedConfig],
258
- device: Optional["torch.device"] = None,
259
- seq_len: Optional[int] = None,
260
- ) -> Tuple["torch.Tensor", float]:
261
- base = config.rope_theta
262
- partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
263
- dim = int((config.hidden_size // config.num_attention_heads) * partial_rotary_factor)
264
-
265
- attention_factor = 1.0 # Unused in this type of RoPE
266
-
267
- inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).float().to(device) / dim))
268
- return inv_freq, attention_factor
269
-
270
- def _compute_linear_scaling_rope_parameters(
271
- self,
272
- config: Optional[PretrainedConfig],
273
- device: Optional["torch.device"] = None,
274
- seq_len: Optional[int] = None,
275
- ) -> Tuple["torch.Tensor", float]:
276
- factor = config.rope_scaling["factor"]
277
- if factor < 1.0:
278
- logger.warning_once(f"`rope_scaling`'s factor field must be a float >= 1, got {factor}")
279
-
280
- inv_freq, attention_factor = self._compute_default_rope_parameters(config, device, seq_len)
281
- inv_freq /= factor
282
- return inv_freq, attention_factor
283
-
284
- def _compute_dynamic_ntk_parameters(
285
- self,
286
- config: Optional[PretrainedConfig],
287
- device: Optional["torch.device"] = None,
288
- seq_len: Optional[int] = None,
289
- ) -> Tuple["torch.Tensor", float]:
290
- base = config.rope_theta
291
- partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
292
- dim = int((config.hidden_size // config.num_attention_heads) * partial_rotary_factor)
293
- max_position_embeddings = config.max_position_embeddings
294
- factor = config.rope_scaling["factor"]
295
- if factor < 1.0:
296
- logger.warning_once(f"`rope_scaling`'s factor field must be a float >= 1, got {factor}")
297
-
298
- attention_factor = 1.0 # Unused in this type of RoPE
299
- seq_len = seq_len if seq_len is not None else max_position_embeddings
300
-
301
- base = base * ((factor * seq_len / max_position_embeddings) - (factor - 1)) ** (dim / (dim - 2))
302
- inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).float().to(device) / dim))
303
- return inv_freq, attention_factor
304
-
305
- def _compute_yarn_parameters(
306
- self,
307
- config: PretrainedConfig,
308
- device: "torch.device",
309
- seq_len: Optional[int] = None,
310
- ) -> Tuple["torch.Tensor", float]:
311
- base = config.rope_theta
312
- partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
313
- dim = int((config.hidden_size // config.num_attention_heads) * partial_rotary_factor)
314
- max_position_embeddings = config.max_position_embeddings
315
- factor = config.rope_scaling["factor"]
316
- if factor < 1.0:
317
- logger.warning_once(f"`rope_scaling`'s factor field must be a float >= 1, got {factor}")
318
-
319
- # Sets the attention factor as suggested in the paper
320
- attention_factor = config.rope_scaling.get("attention_factor")
321
- if attention_factor is None:
322
- attention_factor = 0.1 * math.log(factor) + 1.0
323
- if attention_factor < 0:
324
- logger.warning_once(
325
- f"`rope_scaling`'s attention_factor field must be a float greater than 0, got {attention_factor}"
326
- )
327
-
328
- # Optional config options
329
- # beta_fast/beta_slow: as suggested in the paper, default to 32/1 (correspondingly)
330
- beta_fast = config.rope_scaling.get("beta_fast") or 32
331
- beta_slow = config.rope_scaling.get("beta_slow") or 1
332
- if not isinstance(beta_fast, float):
333
- logger.warning_once(f"`rope_scaling`'s beta_fast field must be a float, got {beta_fast}")
334
- if not isinstance(beta_slow, float):
335
- logger.warning_once(f"`rope_scaling`'s beta_slow field must be a float, got {beta_fast}")
336
- if beta_fast < beta_slow:
337
- logger.warning_once(
338
- f"`rope_scaling`'s beta_fast field must be greater than beta_slow, got beta_fast={beta_fast} "
339
- f"(defaults to 32 if None) and beta_slow={beta_slow} (defaults to 1 if None)"
340
- )
341
-
342
- # Compute the inverse frequencies
343
- def find_correction_dim(num_rotations, dim, base, max_position_embeddings):
344
- """Inverse dimension formula to find the dimension based on the number of rotations"""
345
- return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / (2 * math.log(base))
346
-
347
- def find_correction_range(low_rot, high_rot, dim, base, max_position_embeddings):
348
- """Find dimension range bounds based on rotations"""
349
- low = math.floor(find_correction_dim(low_rot, dim, base, max_position_embeddings))
350
- high = math.ceil(find_correction_dim(high_rot, dim, base, max_position_embeddings))
351
- return max(low, 0), min(high, dim - 1)
352
-
353
- def linear_ramp_mask(min, max, dim):
354
- if min == max:
355
- max += 0.001 # Prevent singularity
356
-
357
- linear_func = (torch.arange(dim, dtype=torch.float32) - min) / (max - min)
358
- ramp_func = torch.clamp(linear_func, 0, 1)
359
- return ramp_func
360
-
361
- pos_freqs = base ** (torch.arange(0, dim, 2).float().to(device) / dim)
362
- inv_freq_extrapolation = 1.0 / pos_freqs
363
- inv_freq_interpolation = 1.0 / (factor * pos_freqs)
364
-
365
- low, high = find_correction_range(beta_fast, beta_slow, dim, base, max_position_embeddings)
366
-
367
- # Get n-dimensional rotational scaling corrected for extrapolation
368
- inv_freq_mask = 1 - linear_ramp_mask(low, high, dim // 2).float().to(device)
369
- inv_freq = inv_freq_interpolation * (1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask
370
-
371
- return inv_freq, attention_factor
372
-
373
- def __init__(self, config: ExaoneConfig, device=None):
374
- ROPE_INIT_FUNCTIONS = {
375
- "default": self._compute_default_rope_parameters,
376
- "linear": self._compute_linear_scaling_rope_parameters,
377
- "dynamic": self._compute_dynamic_ntk_parameters,
378
- "yarn": self._compute_yarn_parameters,
379
- }
380
-
381
- super().__init__()
382
- if config.rope_scaling is not None:
383
- self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
384
- else:
385
- self.rope_type = "default"
386
- self.max_seq_len = config.max_position_embeddings
387
- self.original_max_seq_len = config.max_position_embeddings
388
-
389
- self.config = config
390
- if self.rope_type not in ROPE_INIT_FUNCTIONS:
391
- raise KeyError(f"The EXAONE model does not support RoPE type: {self.rope_type}")
392
- self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
393
-
394
- inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
395
- self.register_buffer("inv_freq", inv_freq, persistent=False)
396
- self.original_inv_freq = self.inv_freq
397
-
398
- def _update_freq(self, position_ids, device):
399
- """
400
- dynamic RoPE layers should recompute `inv_freq` in the following situations:
401
- 1 - growing beyond the cached sequence length (allow scaling)
402
- 2 - the current sequence length is in the original scale (avoid losing precision with small sequences)
403
- """
404
- seq_len = torch.max(position_ids) + 1
405
- if seq_len > self.max_seq_len: # expand to seq_len
406
- inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, seq_len=seq_len)
407
- self.register_buffer("inv_freq", inv_freq, persistent=False)
408
- self.max_seq_len = seq_len
409
-
410
- if seq_len < self.original_max_seq_len and self.max_seq_len > self.original_max_seq_len: # reset to original
411
- self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
412
- self.max_seq_len = self.original_max_seq_len
413
-
414
- @torch.no_grad()
415
- def forward(self, x, position_ids):
416
- if "dynamic" in self.rope_type:
417
- self._update_freq(position_ids, device=x.device)
418
-
419
- inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
420
- position_ids_expanded = position_ids[:, None, :].float()
421
-
422
- device_type = x.device.type
423
- device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
424
- with torch.autocast(device_type=device_type, enabled=False):
425
- freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2)
426
- emb = torch.cat((freqs, freqs), dim=-1)
427
- cos, sin = emb.cos(), emb.sin()
428
-
429
- cos, sin = cos * self.attention_scaling, sin * self.attention_scaling
430
- return cos.to(x.dtype), sin.to(x.dtype)
431
-
432
-
433
- class ExaoneSelfAttention(nn.Module):
434
- def __init__(self, config: ExaoneConfig, layer_idx: Optional[int] = None):
435
- super().__init__()
436
- self.config = config
437
- self.layer_idx = layer_idx
438
- self.embed_dim = config.hidden_size
439
- self.num_heads = config.num_attention_heads
440
- self.head_dim = self.embed_dim // self.num_heads
441
- self.num_key_value_heads = config.num_key_value_heads
442
- self.num_key_value_groups = self.num_heads // self.num_key_value_heads
443
- self.attention_dropout_rate = config.attention_dropout
444
-
445
- if self.head_dim * self.num_heads != self.embed_dim:
446
- raise ValueError(
447
- f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})."
448
- )
449
-
450
- self.rotary = ExaoneRotaryEmbedding(config)
451
-
452
- self.k_proj = nn.Linear(self.embed_dim, self.num_key_value_heads * self.head_dim, bias=False)
453
- self.v_proj = nn.Linear(self.embed_dim, self.num_key_value_heads * self.head_dim, bias=False)
454
- self.q_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=False)
455
- self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
456
-
457
- def forward(
458
- self,
459
- hidden_states: torch.Tensor,
460
- attention_mask: Optional[torch.Tensor] = None,
461
- position_ids: Optional[torch.LongTensor] = None,
462
- past_key_value: Optional[Cache] = None,
463
- output_attentions: Optional[bool] = False,
464
- use_cache: Optional[bool] = False,
465
- cache_position: Optional[torch.LongTensor] = None,
466
- position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
467
- **kwargs,
468
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
469
-
470
- bsz, q_len, _ = hidden_states.size()
471
- query_states = self.q_proj(hidden_states)
472
- key_states = self.k_proj(hidden_states)
473
- value_states = self.v_proj(hidden_states)
474
-
475
- query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
476
- key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
477
- value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
478
-
479
- if position_embeddings is None:
480
- cos, sin = self.rotary(value_states, position_ids=position_ids)
481
- else:
482
- cos, sin = position_embeddings
483
- query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
484
-
485
- if past_key_value is not None:
486
- # sin and cos are specific to RoPE models; cache_position needed for the static cache
487
- cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
488
- key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
489
-
490
- key_states = repeat_kv(key_states, self.num_key_value_groups)
491
- value_states = repeat_kv(value_states, self.num_key_value_groups)
492
-
493
- attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
494
-
495
- if attention_mask is not None:
496
- causal_mask = attention_mask[:, :, :, :key_states.shape[-2]]
497
- attn_weights = attn_weights + causal_mask
498
-
499
- attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
500
- attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout_rate, training=self.training)
501
- attn_output = torch.matmul(attn_weights, value_states)
502
-
503
- if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
504
- raise ValueError(
505
- f"Attention outputs should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
506
- f" {attn_output.size()}"
507
- )
508
-
509
- attn_output = attn_output.transpose(1, 2).contiguous()
510
- attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
511
-
512
- attn_output = self.out_proj(attn_output)
513
-
514
- if not output_attentions:
515
- attn_weights = None
516
-
517
- return attn_output, attn_weights, past_key_value
518
-
519
-
520
- class ExaoneFlashAttention(ExaoneSelfAttention):
521
- def __init__(self, *args, **kwargs):
522
- super().__init__(*args, **kwargs)
523
-
524
- def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
525
- return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
526
-
527
- def forward(
528
- self,
529
- hidden_states: torch.Tensor,
530
- attention_mask: Optional[torch.Tensor] = None,
531
- position_ids: Optional[torch.LongTensor] = None,
532
- past_key_value: Optional[Cache] = None,
533
- output_attentions: Optional[bool] = False,
534
- use_cache: Optional[bool] = False,
535
- cache_position: Optional[torch.LongTensor] = None,
536
- position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
537
- **kwargs,
538
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
539
- if isinstance(past_key_value, StaticCache):
540
- raise ValueError(
541
- "`static` cache implementation is not compatible with `attn_implementation==flash_attention_2` "
542
- "make sure to use `sdpa` in the mean time, and open an issue at https://github.com/huggingface/transformers"
543
- )
544
-
545
- output_attentions = False
546
-
547
- bsz, q_len, h_size = hidden_states.size()
548
-
549
- query_states = self.q_proj(hidden_states)
550
- key_states = self.k_proj(hidden_states)
551
- value_states = self.v_proj(hidden_states)
552
-
553
- query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
554
- key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
555
- value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
556
-
557
- if position_embeddings is None:
558
- cos, sin = self.rotary(value_states, position_ids=position_ids)
559
- else:
560
- cos, sin = position_embeddings
561
- query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
562
-
563
- if past_key_value is not None:
564
- # sin and cos are specific to RoPE models; cache_position needed for the static cache
565
- cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
566
- # Only update cache as shape of [bsz, n_head, q_len, head_dim]
567
- # TODO: need to be fixed when transformers' KV cache layout is changed
568
- key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
569
-
570
- query_states = query_states.transpose(1, 2)
571
- key_states = key_states.transpose(1, 2)
572
- value_states = value_states.transpose(1, 2)
573
-
574
- # In PEFT, usually we cast the layer norms in float32 for training stability reasons
575
- # therefore the input hidden states gets silently casted in float32. Hence, we need
576
- # cast them back in the correct dtype just to be sure everything works as expected.
577
- input_dtype = query_states.dtype
578
- if input_dtype == torch.float32:
579
- if torch.is_autocast_enabled():
580
- target_dtype = torch.get_autocast_gpu_dtype()
581
- # Handle the case where the model is quantized
582
- elif hasattr(self.config, "_pre_quantization_dtype"):
583
- target_dtype = self.config._pre_quantization_dtype
584
- else:
585
- target_dtype = self.q_proj.weight.dtype
586
-
587
- logger.warning_once(
588
- f"The input hidden states seems to be silently casted in float32, this might be related to"
589
- f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
590
- f" {target_dtype}."
591
- )
592
-
593
- query_states = query_states.to(target_dtype)
594
- key_states = key_states.to(target_dtype)
595
- value_states = value_states.to(target_dtype)
596
-
597
- dropout_rate = self.attention_dropout_rate if self.training else 0.0
598
-
599
- attn_output = self._flash_attention_forward(
600
- query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate, is_causal=True
601
- )
602
-
603
- attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
604
- attn_output = self.out_proj(attn_output)
605
-
606
- if not output_attentions:
607
- attn_weights = None
608
-
609
- return attn_output, attn_weights, past_key_value
610
-
611
- @staticmethod
612
- def _flash_attention_forward(
613
- query_states: torch.Tensor,
614
- key_states: torch.Tensor,
615
- value_states: torch.Tensor,
616
- attention_mask: torch.Tensor,
617
- query_length: int,
618
- is_causal: bool,
619
- dropout: float = 0.0,
620
- softmax_scale: Optional[float] = None,
621
- sliding_window: Optional[int] = None,
622
- use_top_left_mask: bool = False,
623
- softcap: Optional[float] = None,
624
- deterministic: bool = os.environ.get("FLASH_ATTENTION_DETERMINISTIC", "0") == "1",
625
- ):
626
- """
627
- Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
628
- first unpad the input, then computes the attention scores and pad the final attention scores.
629
-
630
- Args:
631
- query_states (`torch.Tensor`):
632
- Input query states to be passed to Flash Attention API
633
- key_states (`torch.Tensor`):
634
- Input key states to be passed to Flash Attention API
635
- value_states (`torch.Tensor`):
636
- Input value states to be passed to Flash Attention API
637
- attention_mask (`torch.Tensor`):
638
- The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
639
- position of padding tokens and 1 for the position of non-padding tokens.
640
- dropout (`float`):
641
- Attention dropout
642
- softmax_scale (`float`, *optional*):
643
- The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
644
- use_top_left_mask (`bool`, defaults to `False`):
645
- flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference.
646
- softcap (`float`, *optional*):
647
- Softcap for the attention logits, used e.g. in gemma2.
648
- deterministic (`bool`, *optional*):
649
- Determines if the deterministic option introduced in flash_attn>=2.4.1 is enabled.
650
- """
651
- if not use_top_left_mask:
652
- causal = is_causal
653
- else:
654
- # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__.
655
- causal = is_causal and query_length != 1
656
-
657
- # Assuming 4D tensors, key_states.shape[1] is the key/value sequence length (source length).
658
- use_sliding_windows = (
659
- _flash_supports_window_size and sliding_window is not None and key_states.shape[1] > sliding_window
660
- )
661
- flash_kwargs = {"window_size": (sliding_window, sliding_window)} if use_sliding_windows else {}
662
-
663
- if softcap is not None:
664
- flash_kwargs["softcap"] = softcap
665
-
666
- # Contains at least one padding token in the sequence
667
- if attention_mask is not None:
668
- batch_size = query_states.shape[0]
669
- query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = ExaoneFlashAttention._upad_input(
670
- query_states, key_states, value_states, attention_mask, query_length
671
- )
672
- cu_seqlens_q, cu_seqlens_k = cu_seq_lens
673
- max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
674
-
675
- attn_output_unpad = flash_attn_varlen_func(
676
- query_states,
677
- key_states,
678
- value_states,
679
- cu_seqlens_q=cu_seqlens_q,
680
- cu_seqlens_k=cu_seqlens_k,
681
- max_seqlen_q=max_seqlen_in_batch_q,
682
- max_seqlen_k=max_seqlen_in_batch_k,
683
- dropout_p=dropout,
684
- softmax_scale=softmax_scale,
685
- causal=causal,
686
- **flash_kwargs,
687
- )
688
- attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
689
- else:
690
- attn_output = flash_attn_func(
691
- query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal, **flash_kwargs
692
- )
693
-
694
- return attn_output
695
-
696
- @staticmethod
697
- def _upad_input(
698
- query_layer: torch.Tensor,
699
- key_layer: torch.Tensor,
700
- value_layer: torch.Tensor,
701
- attention_mask: torch.Tensor,
702
- query_length: int,
703
- ):
704
- """
705
- Unpads query, key, and values tensors, using a single dimension for all tokens even though they belong to different batches.
706
-
707
- This function is used instead of `flash_attn.bert_padding.unpad_input` in order to avoid the recomputation of the same intermediary
708
- tensors for query, key, value tensors.
709
-
710
- Arguments:
711
- query_layer (`torch.Tensor`):
712
- Query state with padding. Shape: (batch_size, query_length, num_heads, head_dim).
713
- key_layer (`torch.Tensor`):
714
- Key state with padding. Shape: (batch_size, kv_seq_len, num_key_value_heads, head_dim).
715
- value_layer (`torch.Tensor`):
716
- Value state with padding. Shape: (batch_size, kv_seq_len, num_key_value_heads, head_dim).
717
- attention_mask (`torch.Tensor`):
718
- Boolean or int tensor of shape (batch_size, sequence_length), 1 means valid and 0 means not valid.
719
- query_length (`int`):
720
- Target length.
721
-
722
- Return:
723
- query_layer (`torch.Tensor):
724
- Query state without padding. Shape: (total_target_length, num_heads, head_dim).
725
- key_layer (`torch.Tensor`):
726
- Key state with padding. Shape: (total_source_length, num_key_value_heads, head_dim).
727
- value_layer (`torch.Tensor`):
728
- Value state with padding. Shape: (total_source_length, num_key_value_heads, head_dim).
729
- indices_q (`torch.Tensor`):
730
- The indices of non-masked tokens from the flattened input target sequence.
731
- (cu_seqlens_q, cu_seqlens_k) (`Tuple[int]`):
732
- The cumulative sequence lengths for the target (query) and source (key, value), used to index into ragged (unpadded) tensors. `cu_seqlens` shape is (batch_size + 1,).
733
- (max_seqlen_in_batch_q, max_seqlen_in_batch_k) (`Tuple[int]`):
734
- Maximum sequence length in batch (`max_seqlen_in_batch_q` for the target sequence i.e. query, `max_seqlen_in_batch_k` for the source sequence i.e. key/value).
735
- """
736
- indices_k, cu_seqlens_k, max_seqlen_in_batch_k = ExaoneFlashAttention._get_unpad_data(attention_mask)
737
- batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
738
-
739
- key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k)
740
- value_layer = index_first_axis(
741
- value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
742
- )
743
- if query_length == kv_seq_len:
744
- query_layer = index_first_axis(query_layer.reshape(batch_size * kv_seq_len, -1, head_dim), indices_k)
745
- cu_seqlens_q = cu_seqlens_k
746
- max_seqlen_in_batch_q = max_seqlen_in_batch_k
747
- indices_q = indices_k
748
- elif query_length == 1:
749
- max_seqlen_in_batch_q = 1
750
- cu_seqlens_q = torch.arange(
751
- batch_size + 1, dtype=torch.int32, device=query_layer.device
752
- ) # There is a memcpy here, that is very bad.
753
- indices_q = cu_seqlens_q[:-1]
754
- query_layer = query_layer.squeeze(1)
755
- else:
756
- # The -q_len: slice assumes left padding.
757
- attention_mask = attention_mask[:, -query_length:]
758
- query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
759
-
760
- return (
761
- query_layer,
762
- key_layer,
763
- value_layer,
764
- indices_q,
765
- (cu_seqlens_q, cu_seqlens_k),
766
- (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
767
- )
768
-
769
- @staticmethod
770
- def _get_unpad_data(attention_mask: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, int]:
771
- """
772
- Retrieves indexing data required to repad unpadded (ragged) tensors.
773
-
774
- Arguments:
775
- attention_mask (`torch.Tensor`):
776
- Boolean or int tensor of shape (batch_size, sequence_length), 1 means valid and 0 means not valid.
777
-
778
- Return:
779
- indices (`torch.Tensor):
780
- The indices of non-masked tokens from the flattened input sequence.
781
- cu_seqlens (`torch.Tensor`):
782
- The cumulative sequence lengths, used to index into ragged (unpadded) tensors. `cu_seqlens` shape is (batch_size + 1,).
783
- max_seqlen_in_batch (`int`):
784
- Maximum sequence length in batch.
785
- """
786
- seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
787
- indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
788
- max_seqlen_in_batch = seqlens_in_batch.max().item()
789
- cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
790
- return (
791
- indices,
792
- cu_seqlens,
793
- max_seqlen_in_batch,
794
- )
795
-
796
-
797
- class ExaoneSdpaAttention(ExaoneSelfAttention):
798
- def __init__(self, *args, **kwargs):
799
- super().__init__(*args, **kwargs)
800
-
801
- def forward(
802
- self,
803
- hidden_states: torch.Tensor,
804
- attention_mask: Optional[torch.Tensor] = None,
805
- position_ids: Optional[torch.LongTensor] = None,
806
- past_key_value: Optional[Cache] = None,
807
- output_attentions: Optional[bool] = False,
808
- use_cache: Optional[bool] = False,
809
- cache_position: Optional[torch.LongTensor] = None,
810
- position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
811
- **kwargs,
812
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
813
-
814
- if output_attentions:
815
- logger.warning_once(
816
- "ExaoneModel is using ExaoneSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
817
- 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
818
- )
819
- return super().forward(
820
- hidden_states=hidden_states,
821
- attention_mask=attention_mask,
822
- position_ids=position_ids,
823
- past_key_value=past_key_value,
824
- output_attentions=output_attentions,
825
- use_cache=use_cache,
826
- cache_position=cache_position,
827
- position_embeddings=position_embeddings,
828
- **kwargs,
829
- )
830
-
831
- bsz, q_len, _ = hidden_states.size()
832
-
833
- query_states = self.q_proj(hidden_states)
834
- key_states = self.k_proj(hidden_states)
835
- value_states = self.v_proj(hidden_states)
836
-
837
- query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
838
- key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
839
- value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
840
-
841
- if position_embeddings is None:
842
- cos, sin = self.rotary(value_states, position_ids=position_ids)
843
- else:
844
- cos, sin = position_embeddings
845
- query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
846
-
847
- if past_key_value is not None:
848
- # sin and cos are specific to RoPE models; cache_position needed for the static cache
849
- cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
850
- key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
851
-
852
- key_states = repeat_kv(key_states, self.num_key_value_groups)
853
- value_states = repeat_kv(value_states, self.num_key_value_groups)
854
-
855
- causal_mask = attention_mask
856
- if attention_mask is not None:
857
- causal_mask = causal_mask[:, :, :, :key_states.shape[-2]]
858
-
859
- # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
860
- # Reference: https://github.com/pytorch/pytorch/issues/112577.
861
- if query_states.device.type == "cuda" and causal_mask is not None:
862
- query_states = query_states.contiguous()
863
- key_states = key_states.contiguous()
864
- value_states = value_states.contiguous()
865
-
866
- # We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment
867
- # in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling.
868
- is_causal = True if causal_mask is None and q_len > 1 else False
869
-
870
- attn_output = torch.nn.functional.scaled_dot_product_attention(
871
- query_states,
872
- key_states,
873
- value_states,
874
- attn_mask=causal_mask,
875
- dropout_p=self.attention_dropout_rate if self.training else 0.0,
876
- is_causal=is_causal,
877
- )
878
-
879
- attn_output = attn_output.transpose(1, 2).contiguous()
880
- attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
881
-
882
- attn_output = self.out_proj(attn_output)
883
-
884
- return attn_output, None, past_key_value
885
-
886
-
887
- class ExaoneAttention(nn.Module):
888
- def __init__(self, config, layer_id=0):
889
- super().__init__()
890
- self.layer_id = layer_id
891
- if 'flash' in config._attn_implementation:
892
- self.attention = ExaoneFlashAttention(config, self.layer_id)
893
- elif 'sdpa' in config._attn_implementation:
894
- self.attention = ExaoneSdpaAttention(config, self.layer_id)
895
- else:
896
- self.attention = ExaoneSelfAttention(config, self.layer_id)
897
-
898
- def forward(
899
- self,
900
- hidden_states: torch.Tensor,
901
- attention_mask: Optional[torch.Tensor] = None,
902
- position_ids: Optional[torch.LongTensor] = None,
903
- past_key_value: Optional[Cache] = None,
904
- output_attentions: Optional[bool] = False,
905
- use_cache: Optional[bool] = False,
906
- cache_position: Optional[torch.LongTensor] = None,
907
- position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
908
- **kwargs,
909
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
910
-
911
- return self.attention(
912
- hidden_states=hidden_states,
913
- attention_mask=attention_mask,
914
- position_ids=position_ids,
915
- past_key_value=past_key_value,
916
- output_attentions=output_attentions,
917
- use_cache=use_cache,
918
- cache_position=cache_position,
919
- position_embeddings=position_embeddings,
920
- **kwargs,
921
- )
922
-
923
-
924
- class ExaoneGatedMLP(nn.Module):
925
- def __init__(self, intermediate_size, config):
926
- super().__init__()
927
- self.config = config
928
- embed_dim = config.hidden_size
929
- self.c_fc_0 = nn.Linear(embed_dim, intermediate_size, bias=False)
930
- self.c_fc_1 = nn.Linear(embed_dim, intermediate_size, bias=False)
931
- self.c_proj = nn.Linear(intermediate_size, embed_dim, bias=False)
932
- self.act = ACT2FN[config.activation_function]
933
-
934
- def forward(self, hidden_states):
935
- output_proj = self.c_proj(self.act(self.c_fc_0(hidden_states)) * self.c_fc_1(hidden_states))
936
- return output_proj
937
-
938
-
939
- class ExaoneBlock(nn.Module):
940
- def __init__(self, config, layer_id):
941
- super().__init__()
942
- self.config = config
943
- hidden_size = config.hidden_size
944
- inner_dim = config.intermediate_size if config.intermediate_size is not None else 4 * hidden_size
945
- self.ln_1 = ExaoneRMSNorm(hidden_size = hidden_size, eps=config.layer_norm_epsilon)
946
- self.attn = ExaoneAttention(config, layer_id)
947
- self.ln_2 = ExaoneRMSNorm(hidden_size = hidden_size, eps=config.layer_norm_epsilon)
948
- self.mlp = ExaoneGatedMLP(inner_dim, config)
949
-
950
- def forward(
951
- self,
952
- hidden_states: torch.Tensor,
953
- attention_mask: Optional[torch.Tensor] = None,
954
- position_ids: Optional[torch.LongTensor] = None,
955
- past_key_value: Optional[Cache] = None,
956
- output_attentions: Optional[bool] = False,
957
- use_cache: Optional[bool] = False,
958
- cache_position: Optional[torch.LongTensor] = None,
959
- position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
960
- **kwargs,
961
- ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
962
-
963
- residual = hidden_states
964
- hidden_states = self.ln_1(hidden_states)
965
-
966
- hidden_states, self_attn_weights, present_key_value = self.attn(
967
- hidden_states=hidden_states,
968
- attention_mask=attention_mask,
969
- position_ids=position_ids,
970
- past_key_value=past_key_value,
971
- output_attentions=output_attentions,
972
- use_cache=use_cache,
973
- cache_position=cache_position,
974
- position_embeddings=position_embeddings,
975
- **kwargs,
976
- )
977
- # residual connection
978
- hidden_states = residual + hidden_states
979
-
980
- residual = hidden_states
981
- hidden_states = self.ln_2(hidden_states)
982
- hidden_states = self.mlp(hidden_states)
983
-
984
- hidden_states = residual + hidden_states
985
-
986
- outputs = (hidden_states,)
987
-
988
- if output_attentions:
989
- outputs += (self_attn_weights,)
990
-
991
- if use_cache:
992
- outputs += (present_key_value,)
993
-
994
- return outputs
995
-
996
-
997
- class ExaonePreTrainedModel(PreTrainedModel):
998
- """
999
- An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
1000
- models.
1001
- """
1002
-
1003
- config_class = ExaoneConfig
1004
- base_model_prefix = "transformer"
1005
- supports_gradient_checkpointing = True
1006
- _no_split_modules = ["ExaoneBlock"]
1007
- _skip_keys_device_placement = "past_key_values"
1008
- _supports_flash_attn_2 = True
1009
- _supports_sdpa = True
1010
- _supports_cache_class = True
1011
-
1012
- def __init__(self, *inputs, **kwargs):
1013
- super().__init__(*inputs, **kwargs)
1014
-
1015
- def _init_weights(self, module):
1016
- """Initialize the weights."""
1017
- if isinstance(module, (nn.Linear,)):
1018
- # Slightly different from the TF version which uses truncated_normal for initialization
1019
- # cf https://github.com/pytorch/pytorch/pull/5617
1020
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
1021
- if module.bias is not None:
1022
- module.bias.data.zero_()
1023
- elif isinstance(module, nn.Embedding):
1024
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
1025
- if module.padding_idx is not None:
1026
- module.weight.data[module.padding_idx].zero_()
1027
- elif isinstance(module, ExaoneRMSNorm):
1028
- module.weight.data.fill_(1.0)
1029
-
1030
-
1031
- EXAONE_START_DOCSTRING = r"""
1032
-
1033
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
1034
- library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
1035
- etc.)
1036
-
1037
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
1038
- Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
1039
- and behavior.
1040
-
1041
- Parameters:
1042
- config (:class:`~transformers.ExaoneConfig`): Model configuration class with all the parameters of the model.
1043
- Initializing with a config file does not load the weights associated with the model, only the
1044
- configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
1045
- """
1046
-
1047
- EXAONE_INPUTS_DOCSTRING = r"""
1048
- Args:
1049
- input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, input_ids_length)`):
1050
- :obj:`input_ids_length` = ``sequence_length`` if :obj:`past_key_values` is ``None`` else
1051
- ``past_key_values.get_seq_length()`` (``sequence_length`` of input past key value states). Indices of input
1052
- sequence tokens in the vocabulary.
1053
-
1054
- If :obj:`past_key_values` is used, only ``input_ids`` that do not have their past calculated should be
1055
- passed as ``input_ids``.
1056
-
1057
- `What are input IDs? <../glossary.html#input-ids>`__
1058
- attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
1059
- Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``:
1060
-
1061
- - 1 for tokens that are **not masked**,
1062
- - 0 for tokens that are **masked**.
1063
-
1064
- `What are attention masks? <../glossary.html#attention-mask>`__
1065
- position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
1066
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0,
1067
- config.max_position_embeddings - 1]``.
1068
-
1069
- `What are position IDs? <../glossary.html#position-ids>`_
1070
- past_key_values (:obj:`Cache`, `optional`):
1071
- Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see
1072
- :obj:`past_key_values` output below). Can be used to speed up sequential decoding. This typically consists
1073
- in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or
1074
- `config.use_cache=True`.
1075
- inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
1076
- Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation.
1077
- This is useful if you want more control over how to convert :obj:`input_ids` indices into associated
1078
- vectors than the model's internal embedding lookup matrix.
1079
-
1080
- If :obj:`past_key_values` is used, optionally only the last :obj:`inputs_embeds` have to be input (see
1081
- :obj:`past_key_values`).
1082
- use_cache (:obj:`bool`, `optional`):
1083
- If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
1084
- decoding (see :obj:`past_key_values`).
1085
- output_attentions (:obj:`bool`, `optional`):
1086
- Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
1087
- tensors for more detail.
1088
- output_hidden_states (:obj:`bool`, `optional`):
1089
- Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for
1090
- more detail.
1091
- return_dict (:obj:`bool`, `optional`):
1092
- Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple.
1093
- cache_position (:obj:`torch.LongTensor` of shape :obj:`(sequence_length)`, `optional`):
1094
- Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
1095
- this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
1096
- the complete sequence length.
1097
- """
1098
-
1099
-
1100
- @add_start_docstrings(
1101
- "The bare EXAONE Model transformer outputting raw hidden-states without any specific head on top.",
1102
- EXAONE_START_DOCSTRING,
1103
- )
1104
- class ExaoneModel(ExaonePreTrainedModel):
1105
- def __init__(self, config):
1106
- super().__init__(config)
1107
- self.config = config
1108
- self.embed_dim = config.hidden_size
1109
- self.wte = nn.Embedding(config.vocab_size, self.embed_dim, self.config.pad_token_id)
1110
- self.drop = nn.Dropout(float(config.embed_dropout))
1111
- self.h = nn.ModuleList([ExaoneBlock(config, layer_id=i) for i in range(config.num_layers)])
1112
- self.ln_f = ExaoneRMSNorm(hidden_size=self.embed_dim, eps=config.layer_norm_epsilon)
1113
- self.rotary = ExaoneRotaryEmbedding(config)
1114
- self.gradient_checkpointing = False
1115
- # Initialize weights and apply final processing
1116
- self.post_init()
1117
-
1118
- def get_input_embeddings(self):
1119
- return self.wte
1120
-
1121
- def set_input_embeddings(self, new_embeddings):
1122
- self.wte = new_embeddings
1123
-
1124
- @add_start_docstrings_to_model_forward(EXAONE_INPUTS_DOCSTRING)
1125
- @add_code_sample_docstrings(
1126
- checkpoint=_CHECKPOINT_FOR_DOC,
1127
- output_type=BaseModelOutputWithPastAndCrossAttentions,
1128
- config_class=_CONFIG_FOR_DOC,
1129
- )
1130
- def forward(
1131
- self,
1132
- input_ids: Optional[torch.Tensor] = None,
1133
- attention_mask: Optional[torch.Tensor] = None,
1134
- position_ids: Optional[torch.Tensor] = None,
1135
- past_key_values: Optional[Cache] = None,
1136
- inputs_embeds: Optional[torch.Tensor] = None,
1137
- use_cache: Optional[bool] = None,
1138
- output_attentions: Optional[bool] = None,
1139
- output_hidden_states: Optional[bool] = None,
1140
- return_dict: Optional[bool] = None,
1141
- cache_position: Optional[torch.LongTensor] = None,
1142
- ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPast]:
1143
- output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
1144
- output_hidden_states = (
1145
- output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1146
- )
1147
- use_cache = use_cache if use_cache is not None else self.config.use_cache
1148
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1149
-
1150
- if self.gradient_checkpointing and self.training:
1151
- if use_cache:
1152
- logger.warning_once(
1153
- "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
1154
- )
1155
- use_cache = False
1156
-
1157
- if input_ids is not None and inputs_embeds is not None:
1158
- raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
1159
- elif input_ids is not None:
1160
- batch_size, seq_length = input_ids.shape[:2]
1161
- elif inputs_embeds is not None:
1162
- batch_size, seq_length = inputs_embeds.shape[:2]
1163
- else:
1164
- raise ValueError("You have to specify either input_ids or inputs_embeds")
1165
-
1166
- return_legacy_cache = False
1167
- if (
1168
- use_cache and not isinstance(past_key_values, Cache) and not self.training
1169
- ): # kept for BC (non `Cache` `past_key_values` inputs)
1170
- return_legacy_cache = True
1171
- past_key_values = DynamicCache.from_legacy_cache(past_key_values)
1172
- logger.warning_once(
1173
- "We detected that you are passing `past_key_values` as a tuple and this is deprecated and will be removed in v4.43. "
1174
- "Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/v4.41.3/en/internal/generation_utils#transformers.Cache)"
1175
- )
1176
-
1177
- if inputs_embeds is None:
1178
- inputs_embeds = self.wte(input_ids)
1179
-
1180
- if cache_position is None:
1181
- past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
1182
- cache_position = torch.arange(
1183
- past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
1184
- )
1185
- if position_ids is None:
1186
- position_ids = cache_position.unsqueeze(0)
1187
-
1188
- causal_mask = self._update_causal_mask(
1189
- attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions
1190
- )
1191
-
1192
- hidden_states = inputs_embeds
1193
- hidden_states = self.drop(hidden_states)
1194
-
1195
- position_embeddings = self.rotary(hidden_states, position_ids)
1196
-
1197
- all_hidden_states = () if output_hidden_states else None
1198
- all_self_attns = () if output_attentions else None
1199
- next_decoder_cache = None
1200
-
1201
- for block in self.h:
1202
- if output_hidden_states:
1203
- all_hidden_states = all_hidden_states + (hidden_states,)
1204
-
1205
- if self.gradient_checkpointing and self.training:
1206
- outputs = self._gradient_checkpointing_func(
1207
- block.__call__,
1208
- hidden_states,
1209
- causal_mask,
1210
- position_ids,
1211
- past_key_values,
1212
- output_attentions,
1213
- use_cache,
1214
- cache_position,
1215
- position_embeddings,
1216
- )
1217
- else:
1218
- outputs = block(
1219
- hidden_states,
1220
- attention_mask=causal_mask,
1221
- position_ids=position_ids,
1222
- past_key_value=past_key_values,
1223
- output_attentions=output_attentions,
1224
- use_cache=use_cache,
1225
- cache_position=cache_position,
1226
- position_embeddings=position_embeddings,
1227
- )
1228
-
1229
- hidden_states = outputs[0]
1230
- if use_cache:
1231
- next_decoder_cache = outputs[2 if output_attentions else 1]
1232
-
1233
- if output_attentions:
1234
- all_self_attns += (outputs[1],)
1235
-
1236
- hidden_states = self.ln_f(hidden_states)
1237
- # Add last hidden state
1238
- if output_hidden_states:
1239
- all_hidden_states += (hidden_states,)
1240
-
1241
- next_cache = None
1242
- if use_cache:
1243
- next_cache = next_decoder_cache.to_legacy_cache() if return_legacy_cache else next_decoder_cache
1244
- if not return_dict:
1245
- return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
1246
-
1247
- return BaseModelOutputWithPast(
1248
- last_hidden_state=hidden_states,
1249
- past_key_values=next_cache,
1250
- hidden_states=all_hidden_states,
1251
- attentions=all_self_attns,
1252
- )
1253
-
1254
- # copied from llama
1255
- def _update_causal_mask(
1256
- self,
1257
- attention_mask: torch.Tensor,
1258
- input_tensor: torch.Tensor,
1259
- cache_position: torch.Tensor,
1260
- past_key_values: Cache,
1261
- output_attentions: bool,
1262
- ):
1263
- # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static
1264
- # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes.
1265
- # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using
1266
- # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114
1267
-
1268
- if self.config._attn_implementation == "flash_attention_2":
1269
- if attention_mask is not None and 0.0 in attention_mask:
1270
- return attention_mask
1271
- return None
1272
-
1273
- # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
1274
- # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
1275
- # to infer the attention mask.
1276
- past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
1277
- using_static_cache = isinstance(past_key_values, StaticCache)
1278
-
1279
- # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
1280
- if self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions:
1281
- if AttentionMaskConverter._ignore_causal_mask_sdpa(
1282
- attention_mask,
1283
- inputs_embeds=input_tensor,
1284
- past_key_values_length=past_seen_tokens,
1285
- is_training=self.training,
1286
- ):
1287
- return None
1288
-
1289
- dtype, device = input_tensor.dtype, input_tensor.device
1290
- min_dtype = torch.finfo(dtype).min
1291
- sequence_length = input_tensor.shape[1]
1292
- if using_static_cache:
1293
- target_length = past_key_values.get_max_length()
1294
- else:
1295
- target_length = (
1296
- attention_mask.shape[-1]
1297
- if isinstance(attention_mask, torch.Tensor)
1298
- else past_seen_tokens + sequence_length + 1
1299
- )
1300
-
1301
- # In case the provided `attention` mask is 2D, we generate a causal mask here (4D).
1302
- causal_mask = _prepare_4d_causal_attention_mask_with_cache_position(
1303
- attention_mask,
1304
- sequence_length=sequence_length,
1305
- target_length=target_length,
1306
- dtype=dtype,
1307
- device=device,
1308
- min_dtype=min_dtype,
1309
- cache_position=cache_position,
1310
- batch_size=input_tensor.shape[0],
1311
- )
1312
-
1313
- if (
1314
- self.config._attn_implementation == "sdpa"
1315
- and attention_mask is not None
1316
- and attention_mask.device.type == "cuda"
1317
- and not output_attentions
1318
- ):
1319
- # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
1320
- # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
1321
- # Details: https://github.com/pytorch/pytorch/issues/110213
1322
- causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype)
1323
-
1324
- return causal_mask
1325
-
1326
-
1327
- @add_start_docstrings(
1328
- """
1329
- The EXAONE Model transformer with a language modeling head on top (linear layer with weights tied to the input
1330
- embeddings).
1331
- """,
1332
- EXAONE_START_DOCSTRING,
1333
- )
1334
- class ExaoneForCausalLM(ExaonePreTrainedModel):
1335
-
1336
- def __init__(self, config):
1337
- super().__init__(config)
1338
- self.transformer = ExaoneModel(config)
1339
- self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
1340
- self.config = config
1341
- # Initialize weights and apply final processing
1342
- self.post_init()
1343
-
1344
- def get_output_embeddings(self):
1345
- return self.lm_head
1346
-
1347
- def set_output_embeddings(self, new_embeddings):
1348
- self.lm_head = new_embeddings
1349
-
1350
- @add_start_docstrings_to_model_forward(EXAONE_INPUTS_DOCSTRING)
1351
- @add_code_sample_docstrings(
1352
- checkpoint=_CHECKPOINT_FOR_DOC,
1353
- output_type=BaseModelOutputWithPast,
1354
- config_class=_CONFIG_FOR_DOC,
1355
- )
1356
- def forward(
1357
- self,
1358
- input_ids: Optional[torch.Tensor] = None,
1359
- attention_mask: Optional[torch.Tensor] = None,
1360
- position_ids: Optional[torch.Tensor] = None,
1361
- past_key_values: Optional[Cache] = None,
1362
- inputs_embeds: Optional[torch.Tensor] = None,
1363
- labels: Optional[torch.Tensor] = None,
1364
- use_cache: Optional[bool] = None,
1365
- output_attentions: Optional[bool] = None,
1366
- output_hidden_states: Optional[bool] = None,
1367
- return_dict: Optional[bool] = None,
1368
- cache_position: Optional[torch.LongTensor] = None,
1369
- ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPast]:
1370
- r"""
1371
- Args:
1372
- labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1373
- Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
1374
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
1375
- are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
1376
-
1377
- Example:
1378
-
1379
- ```python
1380
- >>> from transformers import AutoModelForCausalLM, AutoTokenizer
1381
-
1382
- >>> model = AutoModelForCausalLM.from_pretrained("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct",
1383
- trust_remote_code=True)
1384
- >>> tokenizer = AutoTokenizer.from_pretrained("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct")
1385
-
1386
- >>> prompt = "Explain how wonderful you are"
1387
- >>> messages = [
1388
- {"role": "system", "content": "You are a helpful assistant."},
1389
- {"role": "user", "content": prompt}
1390
- ]
1391
- >>> input_ids = tokenizer.apply_chat_template(
1392
- messages,
1393
- tokenize=True,
1394
- add_generation_prompt=True,
1395
- return_tensors="pt"
1396
- )
1397
-
1398
- >>> output = model.generate(input_ids, max_new_tokens=128)
1399
- >>> tokenizer.decode(output[0], skip_special_tokens=True)
1400
- "[|system|]You are a helpful assistant.\n[|user|]Explain how wonderful you are\n[|assistant|]Thank you for your kind words! I'm here to assist you with information, answer questions, and help you in any way I can. My goal is to provide accurate, helpful, and timely responses. Whether you need help with a specific task, want to learn something new, or just need someone to talk to, I'm here for you. How can I assist you today?"
1401
- ```
1402
- """
1403
-
1404
- output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
1405
- output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1406
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1407
- transformer_outputs = self.transformer(
1408
- input_ids,
1409
- attention_mask=attention_mask,
1410
- past_key_values=past_key_values,
1411
- position_ids=position_ids,
1412
- inputs_embeds=inputs_embeds,
1413
- use_cache=use_cache,
1414
- output_attentions=output_attentions,
1415
- output_hidden_states=output_hidden_states,
1416
- return_dict=return_dict,
1417
- cache_position=cache_position,
1418
- )
1419
- hidden_states = transformer_outputs[0]
1420
- lm_logits = self.lm_head(hidden_states)
1421
- lm_logits = lm_logits.float()
1422
- loss = None
1423
- if labels is not None:
1424
- lm_logits = lm_logits.to(torch.float32)
1425
-
1426
- # Shift so that tokens < n predict n
1427
- shift_logits = lm_logits[..., :-1, :].contiguous()
1428
- shift_labels = labels[..., 1:].contiguous()
1429
- # Flatten the tokens
1430
- loss_fct = CrossEntropyLoss()
1431
- loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
1432
-
1433
- lm_logits = lm_logits.to(hidden_states.dtype)
1434
- loss = loss.to(hidden_states.dtype)
1435
-
1436
- if not return_dict:
1437
- output = (lm_logits,) + transformer_outputs[1:]
1438
- return ((loss,) + output) if loss is not None else output
1439
-
1440
- return CausalLMOutputWithPast(
1441
- loss=loss,
1442
- logits=lm_logits,
1443
- past_key_values=transformer_outputs.past_key_values,
1444
- hidden_states=transformer_outputs.hidden_states,
1445
- attentions=transformer_outputs.attentions,
1446
- )
1447
-
1448
- def prepare_inputs_for_generation(
1449
- self,
1450
- input_ids,
1451
- past_key_values=None,
1452
- attention_mask=None,
1453
- inputs_embeds=None,
1454
- cache_position=None,
1455
- position_ids=None,
1456
- use_cache=True,
1457
- **kwargs,
1458
- ):
1459
- # If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens
1460
- # Exception 1: when passing input_embeds, input_ids may be missing entries
1461
- # Exception 2: some generation methods do special slicing of input_ids, so we don't need to do it here
1462
- if past_key_values is not None:
1463
- if inputs_embeds is not None: # Exception 1
1464
- input_ids = input_ids[:, -cache_position.shape[0] :]
1465
- elif input_ids.shape[1] != cache_position.shape[0]: # Default case (the "else", a no op, is Exception 2)
1466
- input_ids = input_ids[:, cache_position]
1467
-
1468
- if attention_mask is not None and position_ids is None:
1469
- # create position_ids on the fly for batch generation
1470
- position_ids = attention_mask.long().cumsum(-1) - 1
1471
- position_ids.masked_fill_(attention_mask == 0, 1)
1472
- if past_key_values:
1473
- position_ids = position_ids[:, -input_ids.shape[1] :]
1474
-
1475
- # This `clone` call is needed to avoid recapturing cuda graphs with `torch.compile`'s `mode="reduce-overhead`, as otherwise the input `position_ids` would have various stride during the decoding. Here, simply using `.contiguous()` is not sufficient as in the batch size = 1 case, `position_ids` is already contiguous but with varying stride which retriggers a capture.
1476
- position_ids = position_ids.clone(memory_format=torch.contiguous_format)
1477
-
1478
- # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
1479
- if inputs_embeds is not None and cache_position[0] == 0:
1480
- model_inputs = {"inputs_embeds": inputs_embeds}
1481
- else:
1482
- model_inputs = {"input_ids": input_ids}
1483
-
1484
- if isinstance(past_key_values, StaticCache) and attention_mask.ndim == 2:
1485
- if inputs_embeds is not None:
1486
- batch_size, sequence_length = inputs_embeds.shape
1487
- device = inputs_embeds.device
1488
- else:
1489
- batch_size, sequence_length = input_ids.shape
1490
- device = input_ids.device
1491
-
1492
- dtype = self.lm_head.weight.dtype
1493
- min_dtype = torch.finfo(dtype).min
1494
-
1495
- attention_mask = _prepare_4d_causal_attention_mask_with_cache_position(
1496
- attention_mask,
1497
- sequence_length=sequence_length,
1498
- target_length=past_key_values.get_max_length(),
1499
- dtype=dtype,
1500
- device=device,
1501
- min_dtype=min_dtype,
1502
- cache_position=cache_position,
1503
- batch_size=batch_size,
1504
- )
1505
-
1506
- model_inputs.update(
1507
- {
1508
- "position_ids": position_ids,
1509
- "cache_position": cache_position,
1510
- "past_key_values": past_key_values,
1511
- "use_cache": use_cache,
1512
- "attention_mask": attention_mask,
1513
- }
1514
- )
1515
- return model_inputs
1516
-
1517
- @staticmethod
1518
- def _reorder_cache(past_key_values, beam_idx):
1519
- reordered_past = ()
1520
- for layer_past in past_key_values:
1521
- reordered_past += (
1522
- tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
1523
- )
1524
- return reordered_past
1525
-
1526
-
1527
- @add_start_docstrings(
1528
- """
1529
- The EXAONE Model transformer with a sequence classification head on top (linear layer).
1530
-
1531
- :class:`~transformers.ExaoneForSequenceClassification` uses the last token in order to do the classification, as
1532
- other causal models (e.g. GPT-1) do.
1533
-
1534
- Since it does classification on the last token, it requires to know the position of the last token. If a
1535
- :obj:`pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
1536
- row. If no :obj:`pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
1537
- guess the padding tokens when :obj:`inputs_embeds` are passed instead of :obj:`input_ids`, it does the same (take
1538
- the last value in each row of the batch).
1539
- """,
1540
- EXAONE_START_DOCSTRING,
1541
- )
1542
- class ExaoneForSequenceClassification(ExaonePreTrainedModel):
1543
- _keys_to_ignore_on_load_missing = ["lm_head.weight"]
1544
- def __init__(self, config):
1545
- super().__init__(config)
1546
- self.num_labels = config.num_labels
1547
- self.transformer = ExaoneModel(config)
1548
- self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
1549
-
1550
- # Initialize weights and apply final processing
1551
- self.post_init()
1552
-
1553
- @add_start_docstrings_to_model_forward(EXAONE_INPUTS_DOCSTRING)
1554
- @add_code_sample_docstrings(
1555
- checkpoint=_CHECKPOINT_FOR_DOC,
1556
- output_type=SequenceClassifierOutputWithPast,
1557
- config_class=_CONFIG_FOR_DOC,
1558
- )
1559
- def forward(
1560
- self,
1561
- input_ids: Optional[torch.Tensor] = None,
1562
- attention_mask: Optional[torch.Tensor] = None,
1563
- position_ids: Optional[torch.Tensor] = None,
1564
- past_key_values: Optional[Cache] = None,
1565
- inputs_embeds: Optional[torch.Tensor] = None,
1566
- labels: Optional[torch.Tensor] = None,
1567
- use_cache: Optional[bool] = None,
1568
- output_attentions: Optional[bool] = None,
1569
- output_hidden_states: Optional[bool] = None,
1570
- return_dict: Optional[bool] = None,
1571
- ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutputWithPast]:
1572
- r"""
1573
- labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1574
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1575
- config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1576
- `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1577
- """
1578
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1579
-
1580
- transformer_outputs = self.transformer(
1581
- input_ids,
1582
- attention_mask=attention_mask,
1583
- position_ids=position_ids,
1584
- past_key_values=past_key_values,
1585
- inputs_embeds=inputs_embeds,
1586
- use_cache=use_cache,
1587
- output_attentions=output_attentions,
1588
- output_hidden_states=output_hidden_states,
1589
- return_dict=return_dict,
1590
- )
1591
- hidden_states = transformer_outputs[0]
1592
- logits = self.score(hidden_states)
1593
-
1594
- if input_ids is not None:
1595
- batch_size, sequence_length = input_ids.shape[:2]
1596
- else:
1597
- batch_size, sequence_length = inputs_embeds.shape[:2]
1598
-
1599
- if self.config.pad_token_id is None and batch_size != 1:
1600
- raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
1601
- if self.config.pad_token_id is None:
1602
- sequence_lengths = -1
1603
- else:
1604
- if input_ids is not None:
1605
- # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
1606
- sequence_lengths = torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1
1607
- sequence_lengths = sequence_lengths % input_ids.shape[-1]
1608
- sequence_lengths = sequence_lengths.to(logits.device)
1609
- else:
1610
- sequence_lengths = -1
1611
- logger.warning(
1612
- f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be "
1613
- "unexpected if using padding tokens in conjunction with `inputs_embeds.`"
1614
- )
1615
-
1616
- pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
1617
-
1618
- loss = None
1619
- if labels is not None:
1620
- labels = labels.to(logits.device)
1621
- if self.config.problem_type is None:
1622
- if self.num_labels == 1:
1623
- self.config.problem_type = "regression"
1624
- elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
1625
- self.config.problem_type = "single_label_classification"
1626
- else:
1627
- self.config.problem_type = "multi_label_classification"
1628
-
1629
- if self.config.problem_type == "regression":
1630
- loss_fct = MSELoss()
1631
- if self.num_labels == 1:
1632
- loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
1633
- else:
1634
- loss = loss_fct(pooled_logits, labels)
1635
- elif self.config.problem_type == "single_label_classification":
1636
- loss_fct = CrossEntropyLoss()
1637
- loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
1638
- elif self.config.problem_type == "multi_label_classification":
1639
- loss_fct = BCEWithLogitsLoss()
1640
- loss = loss_fct(pooled_logits, labels)
1641
- if not return_dict:
1642
- output = (pooled_logits,) + transformer_outputs[1:]
1643
- return ((loss,) + output) if loss is not None else output
1644
-
1645
- return SequenceClassifierOutputWithPast(
1646
- loss=loss,
1647
- logits=pooled_logits,
1648
- past_key_values=transformer_outputs.past_key_values,
1649
- hidden_states=transformer_outputs.hidden_states,
1650
- attentions=transformer_outputs.attentions,
1651
- )
1652
-
1653
-
1654
- @add_start_docstrings(
1655
- """
1656
- The EXAONE Model transformer with a span classification head on top for extractive question-answering tasks like
1657
- SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
1658
- """,
1659
- EXAONE_START_DOCSTRING,
1660
- )
1661
- class ExaoneForQuestionAnswering(ExaonePreTrainedModel):
1662
- _keys_to_ignore_on_load_missing = ["lm_head.weight"]
1663
-
1664
- def __init__(self, config):
1665
- super().__init__(config)
1666
- self.num_labels = config.num_labels
1667
- self.transformer = ExaoneModel(config)
1668
- self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
1669
-
1670
- # Model parallel
1671
- self.model_parallel = False
1672
- self.device_map = None
1673
-
1674
- # Initialize weights and apply final processing
1675
- self.post_init()
1676
-
1677
- def forward(
1678
- self,
1679
- input_ids: Optional[torch.LongTensor] = None,
1680
- attention_mask: Optional[torch.FloatTensor] = None,
1681
- position_ids: Optional[torch.LongTensor] = None,
1682
- past_key_values: Optional[Cache] = None,
1683
- inputs_embeds: Optional[torch.FloatTensor] = None,
1684
- start_positions: Optional[torch.LongTensor] = None,
1685
- end_positions: Optional[torch.LongTensor] = None,
1686
- output_attentions: Optional[bool] = None,
1687
- output_hidden_states: Optional[bool] = None,
1688
- return_dict: Optional[bool] = None,
1689
- ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]:
1690
- r"""
1691
- start_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`):
1692
- Labels for position (index) of the start of the labelled span for computing the token classification loss.
1693
- Positions are clamped to the length of the sequence (:obj:`sequence_length`). Position outside of the
1694
- sequence are not taken into account for computing the loss.
1695
- end_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`):
1696
- Labels for position (index) of the end of the labelled span for computing the token classification loss.
1697
- Positions are clamped to the length of the sequence (:obj:`sequence_length`). Position outside of the
1698
- sequence are not taken into account for computing the loss.
1699
- """
1700
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1701
-
1702
- outputs = self.transformer(
1703
- input_ids,
1704
- attention_mask=attention_mask,
1705
- position_ids=position_ids,
1706
- past_key_values=past_key_values,
1707
- inputs_embeds=inputs_embeds,
1708
- output_attentions=output_attentions,
1709
- output_hidden_states=output_hidden_states,
1710
- return_dict=return_dict,
1711
- )
1712
-
1713
- sequence_output = outputs[0]
1714
-
1715
- logits = self.qa_outputs(sequence_output)
1716
- start_logits, end_logits = logits.split(1, dim=-1)
1717
- start_logits = start_logits.squeeze(-1).contiguous()
1718
- end_logits = end_logits.squeeze(-1).contiguous()
1719
-
1720
- total_loss = None
1721
- if start_positions is not None and end_positions is not None:
1722
- # If we are on multi-GPU, split add a dimension
1723
- if len(start_positions.size()) > 1:
1724
- start_positions = start_positions.squeeze(-1).to(start_logits.device)
1725
- if len(end_positions.size()) > 1:
1726
- end_positions = end_positions.squeeze(-1).to(end_logits.device)
1727
- # sometimes the start/end positions are outside our model inputs, we ignore these terms
1728
- ignored_index = start_logits.size(1)
1729
- start_positions = start_positions.clamp(0, ignored_index)
1730
- end_positions = end_positions.clamp(0, ignored_index)
1731
-
1732
- loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
1733
- start_loss = loss_fct(start_logits, start_positions)
1734
- end_loss = loss_fct(end_logits, end_positions)
1735
- total_loss = (start_loss + end_loss) / 2
1736
-
1737
- if not return_dict:
1738
- output = (start_logits, end_logits) + outputs[2:]
1739
- return ((total_loss,) + output) if total_loss is not None else output
1740
-
1741
- return QuestionAnsweringModelOutput(
1742
- loss=total_loss,
1743
- start_logits=start_logits,
1744
- end_logits=end_logits,
1745
- hidden_states=outputs.hidden_states,
1746
- attentions=outputs.attentions,
1747
- )