Commit
•
2cf1f8d
1
Parent(s):
8c17d87
Create modeling_falcon.py
Browse files- modeling_falcon.py +624 -0
modeling_falcon.py
ADDED
@@ -0,0 +1,624 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# coding=utf-8
|
2 |
+
# Copyright 2023 the Falcon authors and HuggingFace Inc. team. All rights reserved.
|
3 |
+
#
|
4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5 |
+
# you may not use this file except in compliance with the License.
|
6 |
+
# You may obtain a copy of the License at
|
7 |
+
#
|
8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9 |
+
#
|
10 |
+
# Unless required by applicable law or agreed to in writing, software
|
11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13 |
+
# See the License for the specific language governing permissions and
|
14 |
+
# limitations under the License.
|
15 |
+
"""PyTorch Falcon model."""
|
16 |
+
|
17 |
+
import math
|
18 |
+
from typing import Optional, Tuple, Union
|
19 |
+
|
20 |
+
import torch
|
21 |
+
import torch.utils.checkpoint
|
22 |
+
from torch import nn
|
23 |
+
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, LayerNorm, MSELoss
|
24 |
+
from torch.nn import functional as F
|
25 |
+
|
26 |
+
from transformers.modeling_outputs import (
|
27 |
+
BaseModelOutputWithPastAndCrossAttentions,
|
28 |
+
CausalLMOutputWithCrossAttentions,
|
29 |
+
QuestionAnsweringModelOutput,
|
30 |
+
SequenceClassifierOutputWithPast,
|
31 |
+
TokenClassifierOutput,
|
32 |
+
)
|
33 |
+
from transformers.modeling_utils import PreTrainedModel
|
34 |
+
from transformers.utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
|
35 |
+
from .configuration_falcon import FalconConfig
|
36 |
+
|
37 |
+
|
38 |
+
logger = logging.get_logger(__name__)
|
39 |
+
|
40 |
+
FALCON_PRETRAINED_MODEL_ARCHIVE_LIST = [
|
41 |
+
"tiiuae/falcon-40b",
|
42 |
+
"tiiuae/falcon-40b-instruct",
|
43 |
+
"tiiuae/falcon-7b",
|
44 |
+
"tiiuae/falcon-7b-instruct",
|
45 |
+
"tiiuae/falcon-rw-7b",
|
46 |
+
"tiiuae/falcon-rw-1b",
|
47 |
+
]
|
48 |
+
_CHECKPOINT_FOR_DOC = "Rocketknight1/falcon-rw-1b"
|
49 |
+
_CONFIG_FOR_DOC = "FalconConfig"
|
50 |
+
|
51 |
+
|
52 |
+
# NOTE(Hesslow): Unfortunately we did not fuse matmul and bias during training, this means that there's one additional quantization to bfloat16 between the operations.
|
53 |
+
# In order not to degrade the quality of our HF-port, we keep these characteristics in the final model.
|
54 |
+
class FalconLinear(nn.Linear):
|
55 |
+
def forward(self, input: torch.Tensor) -> torch.Tensor:
|
56 |
+
hidden_states = input @ self.weight.T
|
57 |
+
if self.bias is None:
|
58 |
+
return hidden_states
|
59 |
+
return hidden_states + self.bias
|
60 |
+
|
61 |
+
|
62 |
+
# rotary pos emb helpers (torch.jit.script does not seem to support staticmethod...)
|
63 |
+
def rotate_half(x):
|
64 |
+
x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
|
65 |
+
return torch.cat((-x2, x1), dim=-1)
|
66 |
+
|
67 |
+
|
68 |
+
class FalconRotaryEmbedding(nn.Module):
|
69 |
+
"""Implementation of RotaryEmbedding from GPT-NeoX.
|
70 |
+
This implementation is designed to operate on queries and keys that are compatible with `[batch_size,
|
71 |
+
n_heads_per_partition, seq_len, head_dim]` (e.g. MinGPTAttention format).
|
72 |
+
"""
|
73 |
+
|
74 |
+
def __init__(self, head_dim: int, base=10000):
|
75 |
+
super().__init__()
|
76 |
+
inv_freq = 1.0 / (base ** (torch.arange(0, head_dim, 2).float() / head_dim))
|
77 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
78 |
+
self.head_dim = head_dim
|
79 |
+
self.seq_len_cached = -1
|
80 |
+
self.cos_cached: torch.Tensor | None = None
|
81 |
+
self.sin_cached: torch.Tensor | None = None
|
82 |
+
|
83 |
+
def cos_sin(self, seq_len: int, past_key_values_length: int, device="cpu", dtype=torch.bfloat16) -> torch.Tensor:
|
84 |
+
total_length = seq_len + past_key_values_length
|
85 |
+
if total_length > self.seq_len_cached:
|
86 |
+
self.seq_len_cached = total_length
|
87 |
+
t = torch.arange(total_length, device=device, dtype=self.inv_freq.dtype)
|
88 |
+
freqs = torch.einsum("i,j->ij", t, self.inv_freq)
|
89 |
+
emb = torch.cat((freqs, freqs), dim=-1).to(device)
|
90 |
+
|
91 |
+
if dtype in [torch.float16, torch.bfloat16]:
|
92 |
+
emb = emb.float()
|
93 |
+
|
94 |
+
self.cos_cached = emb.cos()[None, :, :]
|
95 |
+
self.sin_cached = emb.sin()[None, :, :]
|
96 |
+
|
97 |
+
self.cos_cached = self.cos_cached.type(dtype)
|
98 |
+
self.sin_cached = self.sin_cached.type(dtype)
|
99 |
+
|
100 |
+
return (
|
101 |
+
self.cos_cached[:, past_key_values_length : seq_len + past_key_values_length],
|
102 |
+
self.sin_cached[:, past_key_values_length : seq_len + past_key_values_length],
|
103 |
+
)
|
104 |
+
|
105 |
+
def forward(self, query, key, past_key_values_length=0):
|
106 |
+
batch, seq_len, head_dim = query.shape
|
107 |
+
cos, sin = self.cos_sin(seq_len, past_key_values_length, query.device, query.dtype)
|
108 |
+
return (query * cos) + (rotate_half(query) * sin), (key * cos) + (rotate_half(key) * sin)
|
109 |
+
|
110 |
+
|
111 |
+
def _make_causal_mask(
|
112 |
+
input_ids_shape: torch.Size, device: torch.device, past_key_values_length: int
|
113 |
+
) -> torch.BoolTensor:
|
114 |
+
"""
|
115 |
+
Make causal mask used for self-attention. This mask does not take the existing attention mask into account - it
|
116 |
+
just blocks tokens from attending forwards in the sequence. The output shape will be `[batch_size, 1,
|
117 |
+
target_length, target_length+past_key_values_length]`.
|
118 |
+
"""
|
119 |
+
batch_size, target_length = input_ids_shape
|
120 |
+
|
121 |
+
mask = torch.triu(torch.ones((target_length, target_length), dtype=torch.bool, device=device), diagonal=1)
|
122 |
+
# If past_key_values_length is 0 this is an empty tensor and the concatenation is a no-op.
|
123 |
+
# This code style is an unfortunate consequence of getting your TF engineer to port models; doing it this
|
124 |
+
# way avoids a data-dependent conditional, which will help me when I have to port this to XLA later.
|
125 |
+
past_mask = torch.zeros((target_length, past_key_values_length), dtype=torch.bool, device=device)
|
126 |
+
mask = torch.cat([past_mask, mask], dim=-1)
|
127 |
+
expanded_mask = mask[None, None, :, :].expand(batch_size, 1, target_length, target_length + past_key_values_length)
|
128 |
+
return expanded_mask
|
129 |
+
|
130 |
+
|
131 |
+
def _expand_mask(mask: torch.Tensor, past_key_values_length: int) -> torch.BoolTensor:
|
132 |
+
"""
|
133 |
+
Expands attention_mask from `[batch_size, seq_length]` to `[batch_size, 1, seq_length, seq_length + past_length]`.
|
134 |
+
"""
|
135 |
+
batch_size, total_length = mask.shape
|
136 |
+
seq_length = total_length - past_key_values_length if past_key_values_length is not None else total_length
|
137 |
+
|
138 |
+
expanded_mask = ~(mask[:, None, None, :].to(torch.bool))
|
139 |
+
return expanded_mask.expand(batch_size, 1, seq_length, total_length)
|
140 |
+
|
141 |
+
|
142 |
+
def build_alibi_tensor(attention_mask: torch.Tensor, num_heads: int, dtype: torch.dtype) -> torch.Tensor:
|
143 |
+
batch_size, seq_length = attention_mask.shape
|
144 |
+
closest_power_of_2 = 2 ** math.floor(math.log2(num_heads))
|
145 |
+
base = torch.tensor(
|
146 |
+
2 ** (-(2 ** -(math.log2(closest_power_of_2) - 3))), device=attention_mask.device, dtype=torch.float32
|
147 |
+
)
|
148 |
+
powers = torch.arange(1, 1 + closest_power_of_2, device=attention_mask.device, dtype=torch.int32)
|
149 |
+
slopes = torch.pow(base, powers)
|
150 |
+
|
151 |
+
if closest_power_of_2 != num_heads:
|
152 |
+
extra_base = torch.tensor(
|
153 |
+
2 ** (-(2 ** -(math.log2(2 * closest_power_of_2) - 3))), device=attention_mask.device, dtype=torch.float32
|
154 |
+
)
|
155 |
+
num_remaining_heads = min(closest_power_of_2, num_heads - closest_power_of_2)
|
156 |
+
extra_powers = torch.arange(1, 1 + 2 * num_remaining_heads, 2, device=attention_mask.device, dtype=torch.int32)
|
157 |
+
slopes = torch.cat([slopes, torch.pow(extra_base, extra_powers)], dim=0)
|
158 |
+
|
159 |
+
# Note: alibi will added to the attention bias that will be applied to the query, key product of attention
|
160 |
+
# => therefore alibi will have to be of shape (batch_size, num_heads, query_length, key_length)
|
161 |
+
# => here we set (batch_size=1, num_heads=num_heads, query_length=1, key_length=max_length)
|
162 |
+
# => the query_length dimension will then be broadcasted correctly
|
163 |
+
# This is more or less identical to T5's relative position bias:
|
164 |
+
# https://github.com/huggingface/transformers/blob/f681437203baa7671de3174b0fa583c349d9d5e1/src/transformers/models/t5/modeling_t5.py#L527
|
165 |
+
arange_tensor = ((attention_mask.cumsum(dim=-1) - 1) * attention_mask)[:, None, :]
|
166 |
+
alibi = slopes[..., None].bfloat16() * arange_tensor
|
167 |
+
return alibi.reshape(batch_size * num_heads, 1, seq_length).to(dtype)
|
168 |
+
|
169 |
+
|
170 |
+
# Copied from transformers.models.bloom.modeling_bloom.dropout_add
|
171 |
+
def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor:
|
172 |
+
"""
|
173 |
+
Dropout add function
|
174 |
+
Args:
|
175 |
+
x (`torch.tensor`, *required*):
|
176 |
+
input tensor
|
177 |
+
residual (`torch.tensor`, *required*):
|
178 |
+
residual tensor
|
179 |
+
prob (`float`, *required*):
|
180 |
+
dropout probability
|
181 |
+
training (`bool`, *required*):
|
182 |
+
training mode
|
183 |
+
"""
|
184 |
+
out = F.dropout(x, p=prob, training=training)
|
185 |
+
out = residual + out
|
186 |
+
return out
|
187 |
+
|
188 |
+
|
189 |
+
class FalconAttention(nn.Module):
|
190 |
+
def __init__(self, config: FalconConfig):
|
191 |
+
super().__init__()
|
192 |
+
|
193 |
+
self.hidden_size = config.hidden_size
|
194 |
+
self.num_heads = config.num_attention_heads
|
195 |
+
self.head_dim = self.hidden_size // self.num_heads
|
196 |
+
self.split_size = self.hidden_size
|
197 |
+
self.hidden_dropout = config.hidden_dropout
|
198 |
+
|
199 |
+
if self.head_dim * self.num_heads != self.hidden_size:
|
200 |
+
raise ValueError(
|
201 |
+
f"`hidden_size` must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:"
|
202 |
+
f" {self.num_heads})."
|
203 |
+
)
|
204 |
+
|
205 |
+
self.maybe_rotary = FalconRotaryEmbedding(config.head_dim) if config.rotary else lambda q, k, t: (q, k)
|
206 |
+
|
207 |
+
# Layer-wise attention scaling
|
208 |
+
self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim)
|
209 |
+
self.beta = self.inv_norm_factor
|
210 |
+
if config.new_decoder_architecture:
|
211 |
+
qkv_out_dim = (config.num_kv_heads * 2 + config.num_attention_heads) * self.head_dim
|
212 |
+
elif config.multi_query:
|
213 |
+
qkv_out_dim = self.hidden_size + 2 * self.head_dim
|
214 |
+
else:
|
215 |
+
qkv_out_dim = 3 * self.hidden_size
|
216 |
+
self.query_key_value = FalconLinear(self.hidden_size, qkv_out_dim, bias=config.bias)
|
217 |
+
self.new_decoder_architecture = config.new_decoder_architecture
|
218 |
+
self.multi_query = config.multi_query
|
219 |
+
self.dense = FalconLinear(self.hidden_size, self.hidden_size, bias=config.bias)
|
220 |
+
self.attention_dropout = nn.Dropout(config.attention_dropout)
|
221 |
+
self.num_kv_heads = config.num_kv_heads if (self.new_decoder_architecture or not self.multi_query) else 1
|
222 |
+
|
223 |
+
def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
224 |
+
"""
|
225 |
+
Split the last dimension into (num_heads, head_dim), results share same memory storage as `fused_qkv`
|
226 |
+
Args:
|
227 |
+
fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim]
|
228 |
+
Returns:
|
229 |
+
query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim]
|
230 |
+
value: [batch_size, seq_length, num_heads, head_dim]
|
231 |
+
"""
|
232 |
+
if self.new_decoder_architecture:
|
233 |
+
batch, seq_len, _ = fused_qkv.shape
|
234 |
+
qkv = fused_qkv.view(batch, seq_len, -1, self.num_heads // self.num_kv_heads + 2, self.head_dim)
|
235 |
+
query = qkv[:, :, :, :-2]
|
236 |
+
key = qkv[:, :, :, [-2]]
|
237 |
+
value = qkv[:, :, :, [-1]]
|
238 |
+
key = torch.broadcast_to(key, query.shape)
|
239 |
+
value = torch.broadcast_to(value, query.shape)
|
240 |
+
|
241 |
+
query, key, value = [x.flatten(2, 3) for x in (query, key, value)]
|
242 |
+
return query, key, value
|
243 |
+
elif not self.multi_query:
|
244 |
+
batch_size, seq_length, three_times_hidden_size = fused_qkv.shape
|
245 |
+
fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads, 3, self.head_dim)
|
246 |
+
return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :]
|
247 |
+
else:
|
248 |
+
batch_size, seq_length, three_times_hidden_size = fused_qkv.shape
|
249 |
+
fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads + 2, self.head_dim)
|
250 |
+
return fused_qkv[..., :-2, :], fused_qkv[..., [-2], :], fused_qkv[..., [-1], :]
|
251 |
+
|
252 |
+
# Copied from transformers.models.bloom.modeling_bloom.BloomAttention._merge_heads
|
253 |
+
def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
|
254 |
+
"""
|
255 |
+
Merge heads together over the last dimenstion
|
256 |
+
Args:
|
257 |
+
x (`torch.tensor`, *required*): [batch_size * num_heads, seq_length, head_dim]
|
258 |
+
Returns:
|
259 |
+
torch.tensor: [batch_size, seq_length, num_heads * head_dim]
|
260 |
+
"""
|
261 |
+
# What we want to achieve is:
|
262 |
+
# batch_size * num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads * head_dim
|
263 |
+
batch_size_and_num_heads, seq_length, _ = x.shape
|
264 |
+
batch_size = batch_size_and_num_heads // self.num_heads
|
265 |
+
|
266 |
+
# First view to decompose the batch size
|
267 |
+
# batch_size * num_heads, seq_length, head_dim -> batch_size, num_heads, seq_length, head_dim
|
268 |
+
x = x.view(batch_size, self.num_heads, seq_length, self.head_dim)
|
269 |
+
|
270 |
+
# batch_size, num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads, head_dim
|
271 |
+
x = x.permute(0, 2, 1, 3)
|
272 |
+
|
273 |
+
# batch_size, seq_length, num_heads, head_dim -> batch_size, seq_length, num_heads * head_dim
|
274 |
+
return x.reshape(batch_size, seq_length, self.num_heads * self.head_dim)
|
275 |
+
|
276 |
+
def forward(
|
277 |
+
self,
|
278 |
+
hidden_states: torch.Tensor,
|
279 |
+
alibi: Optional[torch.Tensor],
|
280 |
+
attention_mask: torch.Tensor,
|
281 |
+
layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
282 |
+
head_mask: Optional[torch.Tensor] = None,
|
283 |
+
use_cache: bool = False,
|
284 |
+
output_attentions: bool = False,
|
285 |
+
):
|
286 |
+
fused_qkv = self.query_key_value(hidden_states) # [batch_size, seq_length, 3 x hidden_size]
|
287 |
+
num_kv_heads = self.num_heads if self.new_decoder_architecture else self.num_kv_heads
|
288 |
+
# 3 x [batch_size, seq_length, num_heads, head_dim]
|
289 |
+
(query_layer, key_layer, value_layer) = self._split_heads(fused_qkv)
|
290 |
+
|
291 |
+
batch_size, query_length, _, _ = query_layer.shape
|
292 |
+
|
293 |
+
query_layer = query_layer.transpose(1, 2).reshape(batch_size * self.num_heads, query_length, self.head_dim)
|
294 |
+
key_layer = key_layer.transpose(1, 2).reshape(
|
295 |
+
batch_size * num_kv_heads,
|
296 |
+
query_length,
|
297 |
+
self.head_dim,
|
298 |
+
)
|
299 |
+
value_layer = value_layer.transpose(1, 2).reshape(batch_size * num_kv_heads, query_length, self.head_dim)
|
300 |
+
|
301 |
+
past_kv_length = 0 if layer_past is None else layer_past[0].shape[1]
|
302 |
+
query_layer, key_layer = self.maybe_rotary(query_layer, key_layer, past_kv_length)
|
303 |
+
|
304 |
+
if layer_past is not None:
|
305 |
+
past_key, past_value = layer_past
|
306 |
+
# concatenate along seq_length dimension:
|
307 |
+
# - key: [batch_size * self.num_heads, kv_length, head_dim]
|
308 |
+
# - value: [batch_size * self.num_heads, kv_length, head_dim]
|
309 |
+
key_layer = torch.cat((past_key, key_layer), dim=1)
|
310 |
+
value_layer = torch.cat((past_value, value_layer), dim=1)
|
311 |
+
|
312 |
+
_, kv_length, _ = key_layer.shape
|
313 |
+
if use_cache:
|
314 |
+
present = (key_layer, value_layer)
|
315 |
+
else:
|
316 |
+
present = None
|
317 |
+
|
318 |
+
attention_mask_float = (attention_mask * 1.0).masked_fill(attention_mask, float("-1e9")).to(query_layer.dtype)
|
319 |
+
|
320 |
+
query_layer_ = query_layer.reshape(batch_size, self.num_heads, -1, self.head_dim)
|
321 |
+
key_layer_ = key_layer.reshape(batch_size, num_kv_heads, -1, self.head_dim)
|
322 |
+
value_layer_ = value_layer.reshape(batch_size, num_kv_heads, -1, self.head_dim)
|
323 |
+
|
324 |
+
if alibi is None:
|
325 |
+
if output_attentions:
|
326 |
+
# F.scaled_dot_product_attention doesn't return the attention weights, so we have
|
327 |
+
# to do it by hand if we want them
|
328 |
+
attention_scores = query_layer_ @ key_layer_.transpose(-1, -2)
|
329 |
+
attention_scores /= math.sqrt(self.head_dim)
|
330 |
+
|
331 |
+
attention_scores = F.softmax(
|
332 |
+
attention_scores + attention_mask_float, dim=-1, dtype=hidden_states.dtype
|
333 |
+
)
|
334 |
+
attn_output = attention_scores @ value_layer_
|
335 |
+
else:
|
336 |
+
attn_output = F.scaled_dot_product_attention(
|
337 |
+
query_layer_, key_layer_, value_layer_, attention_mask_float, 0.0, is_causal=False
|
338 |
+
)
|
339 |
+
attention_scores = None
|
340 |
+
|
341 |
+
attn_output = attn_output.view(batch_size, self.num_heads, query_length, self.head_dim)
|
342 |
+
attn_output = attn_output.permute(0, 2, 1, 3)
|
343 |
+
attn_output = attn_output.reshape(batch_size, query_length, self.num_heads * self.head_dim)
|
344 |
+
|
345 |
+
output_tensor = self.dense(attn_output)
|
346 |
+
|
347 |
+
if output_attentions:
|
348 |
+
return output_tensor, present, attention_scores
|
349 |
+
else:
|
350 |
+
return output_tensor, present
|
351 |
+
|
352 |
+
else:
|
353 |
+
matmul_result = query_layer_ @ key_layer_.transpose(-1, -2)
|
354 |
+
|
355 |
+
# change view to [batch_size, num_heads, q_length, kv_length]
|
356 |
+
attention_scores = matmul_result.view(batch_size, self.num_heads, query_length, kv_length)
|
357 |
+
|
358 |
+
# cast attention scores to fp32, compute scaled softmax and cast back to initial dtype - [batch_size, num_heads, q_length, kv_length]
|
359 |
+
input_dtype = attention_scores.dtype
|
360 |
+
# `float16` has a minimum value of -65504.0, whereas `bfloat16` and `float32` have a minimum value of `-3.4e+38`
|
361 |
+
if input_dtype == torch.float16 or input_dtype == torch.bfloat16:
|
362 |
+
attention_scores = attention_scores.to(torch.float32)
|
363 |
+
# Matt (HF) note: We could possibly use F.scaled_dot_product_attention here too, by
|
364 |
+
# adding (alibi * self.inv_norm_factor) to attention_mask_float. I think this would be mathematically
|
365 |
+
# equivalent and more performant, but there might be a numerical difference. If you're reading this
|
366 |
+
# and you'd like to experiment and maybe file a PR, feel free!
|
367 |
+
attention_logits = attention_scores + alibi.view(batch_size, self.num_heads, 1, -1)
|
368 |
+
attention_logits *= self.inv_norm_factor
|
369 |
+
attention_probs = F.softmax(attention_logits + attention_mask_float, dim=-1, dtype=hidden_states.dtype)
|
370 |
+
# [batch_size, num_heads, q_length, kv_length]
|
371 |
+
attention_probs = self.attention_dropout(attention_probs)
|
372 |
+
|
373 |
+
if head_mask is not None:
|
374 |
+
attention_probs = attention_probs * head_mask
|
375 |
+
|
376 |
+
# change view [batch_size, num_heads, q_length, kv_length]
|
377 |
+
attention_probs_reshaped = attention_probs.view(batch_size, self.num_heads, query_length, kv_length)
|
378 |
+
|
379 |
+
# matmul: [batch_size * num_heads, q_length, head_dim]
|
380 |
+
context_layer = (attention_probs_reshaped @ value_layer_).flatten(0, 1)
|
381 |
+
|
382 |
+
# change view [batch_size, num_heads, q_length, head_dim]
|
383 |
+
context_layer = self._merge_heads(context_layer)
|
384 |
+
|
385 |
+
output_tensor = self.dense(context_layer)
|
386 |
+
|
387 |
+
if output_attentions:
|
388 |
+
return output_tensor, present, attention_probs
|
389 |
+
else:
|
390 |
+
return output_tensor, present
|
391 |
+
|
392 |
+
|
393 |
+
class FalconMLP(nn.Module):
|
394 |
+
def __init__(self, config: FalconConfig):
|
395 |
+
super().__init__()
|
396 |
+
hidden_size = config.hidden_size
|
397 |
+
|
398 |
+
self.dense_h_to_4h = FalconLinear(hidden_size, 4 * hidden_size, bias=config.bias)
|
399 |
+
self.act = nn.GELU()
|
400 |
+
self.dense_4h_to_h = FalconLinear(4 * hidden_size, hidden_size, bias=config.bias)
|
401 |
+
self.hidden_dropout = config.hidden_dropout
|
402 |
+
|
403 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
404 |
+
x = self.act(self.dense_h_to_4h(x))
|
405 |
+
x = self.dense_4h_to_h(x)
|
406 |
+
return x
|
407 |
+
|
408 |
+
|
409 |
+
class FalconDecoderLayer(nn.Module):
|
410 |
+
def __init__(self, config: FalconConfig):
|
411 |
+
super().__init__()
|
412 |
+
hidden_size = config.hidden_size
|
413 |
+
self.num_heads = config.num_attention_heads
|
414 |
+
self.self_attention = FalconAttention(config)
|
415 |
+
self.mlp = FalconMLP(config)
|
416 |
+
self.hidden_dropout = config.hidden_dropout
|
417 |
+
self.config = config
|
418 |
+
|
419 |
+
if config.new_decoder_architecture:
|
420 |
+
# The layer norm before self-attention
|
421 |
+
self.ln_attn = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
|
422 |
+
# The layer norm before the MLP
|
423 |
+
self.ln_mlp = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
|
424 |
+
else:
|
425 |
+
self.input_layernorm = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
|
426 |
+
if not config.parallel_attn:
|
427 |
+
self.post_attention_layernorm = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
|
428 |
+
|
429 |
+
def forward(
|
430 |
+
self,
|
431 |
+
hidden_states: torch.Tensor,
|
432 |
+
alibi: Optional[torch.Tensor],
|
433 |
+
attention_mask: torch.Tensor,
|
434 |
+
layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
435 |
+
head_mask: Optional[torch.Tensor] = None,
|
436 |
+
use_cache: bool = False,
|
437 |
+
output_attentions: bool = False,
|
438 |
+
):
|
439 |
+
residual = hidden_states
|
440 |
+
|
441 |
+
if self.config.new_decoder_architecture:
|
442 |
+
attention_layernorm_out = self.ln_attn(hidden_states)
|
443 |
+
mlp_layernorm_out = self.ln_mlp(hidden_states)
|
444 |
+
else:
|
445 |
+
attention_layernorm_out = self.input_layernorm(hidden_states)
|
446 |
+
|
447 |
+
# Self attention.
|
448 |
+
attn_outputs = self.self_attention(
|
449 |
+
attention_layernorm_out,
|
450 |
+
layer_past=layer_past,
|
451 |
+
attention_mask=attention_mask,
|
452 |
+
alibi=alibi,
|
453 |
+
head_mask=head_mask,
|
454 |
+
use_cache=use_cache,
|
455 |
+
output_attentions=output_attentions,
|
456 |
+
)
|
457 |
+
|
458 |
+
attention_output = attn_outputs[0]
|
459 |
+
|
460 |
+
if not self.config.new_decoder_architecture:
|
461 |
+
if self.config.parallel_attn:
|
462 |
+
mlp_layernorm_out = attention_layernorm_out
|
463 |
+
else:
|
464 |
+
residual = dropout_add(
|
465 |
+
attention_output, residual, self.config.attention_dropout, training=self.training
|
466 |
+
)
|
467 |
+
mlp_layernorm_out = self.post_attention_layernorm(residual)
|
468 |
+
|
469 |
+
outputs = attn_outputs[1:]
|
470 |
+
|
471 |
+
# MLP.
|
472 |
+
mlp_output = self.mlp(mlp_layernorm_out)
|
473 |
+
|
474 |
+
if self.config.new_decoder_architecture or self.config.parallel_attn:
|
475 |
+
mlp_output += attention_output
|
476 |
+
|
477 |
+
output = dropout_add(mlp_output, residual, self.config.hidden_dropout, training=self.training)
|
478 |
+
|
479 |
+
if use_cache:
|
480 |
+
outputs = (output,) + outputs
|
481 |
+
else:
|
482 |
+
outputs = (output,) + outputs[1:]
|
483 |
+
|
484 |
+
return outputs # hidden_states, present, attentions
|
485 |
+
|
486 |
+
|
487 |
+
FALCON_START_DOCSTRING = r"""
|
488 |
+
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
489 |
+
library implements for all its model (such as downloading or saving, resizing the input embeddings etc.)
|
490 |
+
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
491 |
+
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
492 |
+
and behavior.
|
493 |
+
Parameters:
|
494 |
+
config ([`FalconConfig`]): Model configuration class with all the parameters of the model.
|
495 |
+
Initializing with a config file does not load the weights associated with the model, only the
|
496 |
+
configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
|
497 |
+
"""
|
498 |
+
|
499 |
+
FALCON_INPUTS_DOCSTRING = r"""
|
500 |
+
Args:
|
501 |
+
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
|
502 |
+
`input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[2]`
|
503 |
+
(`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.
|
504 |
+
If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
|
505 |
+
`input_ids`.
|
506 |
+
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
507 |
+
[`PreTrainedTokenizer.__call__`] for details.
|
508 |
+
[What are input IDs?](../glossary#input-ids)
|
509 |
+
past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.num_hidden_layers`):
|
510 |
+
Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see
|
511 |
+
`past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have
|
512 |
+
their past given to this model should not be passed as `input_ids` as they have already been computed.
|
513 |
+
Each element of `past_key_values` is a tuple (past_key, past_value):
|
514 |
+
- past_key: [batch_size * num_heads, head_dim, kv_length]
|
515 |
+
- past_value: [batch_size * num_heads, kv_length, head_dim]
|
516 |
+
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
517 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
518 |
+
- 1 for tokens that are **not masked**,
|
519 |
+
- 0 for tokens that are **masked**.
|
520 |
+
[What are attention masks?](../glossary#attention-mask)
|
521 |
+
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
|
522 |
+
Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
|
523 |
+
- 1 indicates the head is **not masked**,
|
524 |
+
- 0 indicates the head is **masked**.
|
525 |
+
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
|
526 |
+
Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
|
527 |
+
is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
|
528 |
+
model's internal embedding lookup matrix.
|
529 |
+
If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see
|
530 |
+
`past_key_values`).
|
531 |
+
use_cache (`bool`, *optional*):
|
532 |
+
If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
|
533 |
+
`past_key_values`).
|
534 |
+
output_attentions (`bool`, *optional*):
|
535 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
536 |
+
tensors for more detail.
|
537 |
+
output_hidden_states (`bool`, *optional*):
|
538 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
539 |
+
more detail.
|
540 |
+
return_dict (`bool`, *optional*):
|
541 |
+
Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
|
542 |
+
"""
|
543 |
+
|
544 |
+
|
545 |
+
class FalconPreTrainedModel(PreTrainedModel):
|
546 |
+
"""
|
547 |
+
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
|
548 |
+
models.
|
549 |
+
"""
|
550 |
+
|
551 |
+
config_class = FalconConfig
|
552 |
+
base_model_prefix = "transformer"
|
553 |
+
supports_gradient_checkpointing = True
|
554 |
+
_no_split_modules = ["FalconDecoderLayer"]
|
555 |
+
|
556 |
+
def __init__(self, *inputs, **kwargs):
|
557 |
+
super().__init__(*inputs, **kwargs)
|
558 |
+
|
559 |
+
def _init_weights(self, module: nn.Module):
|
560 |
+
"""Initialize the weights."""
|
561 |
+
if isinstance(module, nn.Linear) or isinstance(module, FalconLinear):
|
562 |
+
# Slightly different from the TF version which uses truncated_normal for initialization
|
563 |
+
# cf https://github.com/pytorch/pytorch/pull/5617
|
564 |
+
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
|
565 |
+
if module.bias is not None:
|
566 |
+
module.bias.data.zero_()
|
567 |
+
elif isinstance(module, nn.Embedding):
|
568 |
+
module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
|
569 |
+
if module.padding_idx is not None:
|
570 |
+
module.weight.data[module.padding_idx].zero_()
|
571 |
+
elif isinstance(module, LayerNorm):
|
572 |
+
module.bias.data.zero_()
|
573 |
+
module.weight.data.fill_(1.0)
|
574 |
+
|
575 |
+
# Copied from transformers.models.bloom.modeling_bloom.BloomPreTrainedModel._set_gradient_checkpointing with BloomModel->FalconModel
|
576 |
+
def _set_gradient_checkpointing(self, module: nn.Module, value: bool = False):
|
577 |
+
if isinstance(module, FalconModel):
|
578 |
+
module.gradient_checkpointing = value
|
579 |
+
|
580 |
+
@staticmethod
|
581 |
+
def _convert_cache_to_standard_format(
|
582 |
+
past_key_value: Tuple[Tuple[torch.Tensor, torch.Tensor]], batch_size: int
|
583 |
+
) -> Tuple[Tuple[torch.Tensor, torch.Tensor]]:
|
584 |
+
"""
|
585 |
+
Standardizes the format of the cache so as to match most implementations, i.e. to tuple(tuple([batch_size,
|
586 |
+
num_heads, ...]))
|
587 |
+
"""
|
588 |
+
batch_size_times_num_heads, kv_length, head_dim = past_key_value[0][0].shape
|
589 |
+
# [batch_size * self.num_heads, kv_length, head_dim] -> [batch_size, num_heads, kv_length, head_dim]
|
590 |
+
# Note that don't want to use self.num_attention_heads because the number of heads may vary depending
|
591 |
+
# on whether we use multi_query attention.
|
592 |
+
num_heads = batch_size_times_num_heads // batch_size
|
593 |
+
return tuple(
|
594 |
+
(
|
595 |
+
layer_past[0].view(batch_size, num_heads, kv_length, head_dim),
|
596 |
+
layer_past[1].view(batch_size, num_heads, kv_length, head_dim),
|
597 |
+
)
|
598 |
+
for layer_past in past_key_value
|
599 |
+
)
|
600 |
+
|
601 |
+
@staticmethod
|
602 |
+
def _convert_to_rw_cache(
|
603 |
+
past_key_value: Tuple[Tuple[torch.Tensor, torch.Tensor]]
|
604 |
+
) -> Tuple[Tuple[torch.Tensor, torch.Tensor]]:
|
605 |
+
batch_size, num_heads, kv_length, head_dim = past_key_value[0][0].shape
|
606 |
+
batch_size_times_num_heads = batch_size * num_heads
|
607 |
+
# [batch_size, num_heads, kv_length, head_dim] -> [batch_size * num_heads, kv_length, head_dim]
|
608 |
+
return tuple(
|
609 |
+
(
|
610 |
+
layer_past[0].view(batch_size_times_num_heads, kv_length, head_dim),
|
611 |
+
layer_past[1].view(batch_size_times_num_heads, kv_length, head_dim),
|
612 |
+
)
|
613 |
+
for layer_past in past_key_value
|
614 |
+
)
|
615 |
+
|
616 |
+
|
617 |
+
@add_start_docstrings(
|
618 |
+
"The bare Falcon Model transformer outputting raw hidden-states without any specific head on top.",
|
619 |
+
FALCON_START_DOCSTRING,
|
620 |
+
)
|
621 |
+
class FalconModel(FalconPreTrainedModel):
|
622 |
+
def __init__(self, config: FalconConfig):
|
623 |
+
super().__init__(config)
|
624 |
+
|