vltnmmdv commited on
Commit
52b380b
1 Parent(s): b8bfbc0

Add files using upload-large-folder tool

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,3 +1,106 @@
1
- ---
2
- license: mit
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: mit
3
+ language:
4
+ - ru
5
+ - en
6
+ ---
7
+
8
+ # GigaChat-20B-A3B-base
9
+
10
+ Большая языковая модель, основанна на MoE архитектуре, обучена специально под русский язык **с нуля**.
11
+ Всего у модели 20 миллиардов параметров, но во время инференса задействовано только 3 миллиарда. Контекст модели =131k токенов.
12
+
13
+ Больше подробностей в [хабр статье](https://habr.com/en/companies/sberdevices/articles/865996/).
14
+
15
+ ## Архитектура модели
16
+
17
+ GigaChat-20B-A3B состоит из следующих деталей:
18
+
19
+ - Fine-grained Experts + Shared Experts
20
+ - Grouped Query Attention
21
+ - Rotary Position Embeddings
22
+ - RMSNorm
23
+ - SwiGLU в MLP
24
+
25
+ Важно то, что в реализации MoE некоторые эксперты вызываются в зависимости от контекста, а другие используются всегда.
26
+
27
+ ## Бенчмарки
28
+
29
+ Общие английские метрики. Для замера использовался популярный открытый репозиторий [LM Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness).
30
+
31
+ | Bench | T-lite-0.1<br>(llama 3.1 8B based)| Llama-3.1-8B | GigaChat-20B-A3B-base | Gemma-9B |
32
+ | ----------------------------- | ---------- | ------------ | --------------------- | --------- |
33
+ | MMLU (5-shot) | 62.56 | 65.21 | 63.02 | 70.6 |
34
+ | MMLU-pro (5-shot) | 32.19 | 35.7 | 31.41 | 42.85 |
35
+ | MMLU-ru (5-shot) | 55.51 | 54.1 | 58.38 | 62.57 |
36
+ | BBH (3-shot) | 62.36 | 62.79 | 53.54 | 70.48 |
37
+ | ARC-C (25-shot) | 58.19 | 54.69 | 61.69 | 68.34 |
38
+ | TruthfulQA (0-shot) (rougeL) | 46.51 | 34.52 | 31.82 | 41.49 |
39
+ | Winogrande (5-shot) | 78.45 | 77.43 | 75.85 | 79.4 |
40
+ | Hellaswag (10-shot) | 82.21 | 81.85 | 81.91 | 82.5 |
41
+ | GPQA (5-shot) | 0.25 | 23.44 | 25.22 | 30.36 |
42
+ | MATH (4-shot) | 12.9 | 14.04 | 15.04 | 20.06 |
43
+ | GSM8K (4-shot) (strict-match) | 67.93 | 51.4 | 59.06 | 68.99 |
44
+ | HumanEval | 16.46 | 25.61 | 32.32 | 37.2 |
45
+ | **AVG** | **47.96** | **48.4** | **49.11** | **56.24** |
46
+
47
+
48
+ ## Requirements
49
+
50
+ * ```transformers>=4.47```
51
+
52
+
53
+ ## Пример использования через transformers
54
+
55
+ ```python
56
+ import torch
57
+ from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
58
+
59
+ model_name = "ai-sage/GigaChat-20B-A3B-base"
60
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
61
+ model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype=torch.bfloat16, device_map="auto")
62
+ model.generation_config = GenerationConfig.from_pretrained(model_name)
63
+
64
+ messages = (
65
+ "Ниже я написал подробное доказательство теоремы о неподвижной точке:"
66
+ )
67
+ input_tensor = tokenizer(messages, return_tensors="pt").input_ids
68
+ outputs = model.generate(input_tensor.to(model.device))
69
+
70
+ result = tokenizer.decode(outputs[0][input_tensor.shape[1]:], skip_special_tokens=False)
71
+ print(result)
72
+ ```
73
+
74
+ ## Пример использования через vLLM
75
+
76
+ ```python
77
+ from transformers import AutoTokenizer
78
+ from vllm import LLM, SamplingParams
79
+
80
+ model_name = "ai-sage/GigaChat-20B-A3B-base"
81
+ llm = LLM(model=model_name, tokenizer=model_name, trust_remote_code=True)
82
+ sampling_params = SamplingParams(
83
+ temperature=0.3,
84
+ max_tokens=8192,
85
+ stop_token_ids=[tokenizer.eos_token_id]
86
+ )
87
+
88
+ messages = (
89
+ "Ниже я написал подробное доказательство теоремы о неподвижной точке:"
90
+ )
91
+ outputs = llm.generate(messages, sampling_params=sampling_params)
92
+ generated_text = [output.outputs[0].text for output in outputs]
93
+ print(generated_text)
94
+ ```
95
+
96
+ ## Скорость генерации
97
+
98
+ | Model | Total params (B) | Active params (B) | Req/s | Output Token/s | Total Token/s |
99
+ |---------|-----------------|------------------|--------|----------------|----------------|
100
+ | Qwen/Qwen1.5-MoE-A2.7B-Chat | 14 | 2,7 | 0,62 | 156,43 | 291,17 |
101
+ | deepseek-ai/deepseek-moe-16b-chat | 16 | 2,8 | 0,59 | 149,53 | 285,39 |
102
+ | **GigaChat-20B-A3B** | 20 | 3,3 | 0,55 | 137,43 | 259,27 |
103
+ | Qwen/Qwen2.5-3B-Instruct | 3 | 3 | 0,54 | 135,10 | 251,44 |
104
+ | meta-llama/Meta-Llama-3-8B-Instruct | 8 | 8 | 0,35 | 83,26 | 157,32 |
105
+ | google/gemma-2-9b-it | 9 | 9 | 0,27 | 54,87 | 113,69 |
106
+
config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "DeepseekForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_deepseek.DeepseekConfig",
9
+ "AutoModel": "modelling_deepseek.DeepseekModel",
10
+ "AutoModelForCausalLM": "modelling_deepseek.DeepseekForCausalLM"
11
+ },
12
+ "aux_loss_alpha": 0.001,
13
+ "bos_token_id": 1,
14
+ "eos_token_id": 2,
15
+ "first_k_dense_replace": 1,
16
+ "head_dim": 128,
17
+ "hidden_act": "silu",
18
+ "hidden_size": 2048,
19
+ "initializer_range": 0.006,
20
+ "intermediate_size": 14336,
21
+ "max_position_embeddings": 131072,
22
+ "mlp_bias": false,
23
+ "model_type": "deepseek",
24
+ "moe_implementation": "eager",
25
+ "moe_intermediate_size": 1792,
26
+ "moe_layer_freq": 1,
27
+ "n_routed_experts": 64,
28
+ "n_shared_experts": 2,
29
+ "norm_topk_prob": false,
30
+ "num_attention_heads": 16,
31
+ "num_experts_per_tok": 6,
32
+ "num_hidden_layers": 28,
33
+ "num_key_value_heads": 8,
34
+ "pad_token_id": 2,
35
+ "pretraining_tp": 1,
36
+ "rms_norm_eps": 1e-05,
37
+ "rope_scaling": null,
38
+ "rope_theta": 1400000,
39
+ "scoring_func": "softmax",
40
+ "seq_aux": true,
41
+ "tie_word_embeddings": false,
42
+ "torch_dtype": "float32",
43
+ "transformers_version": "4.47.0",
44
+ "use_cache": true,
45
+ "vocab_size": 128256
46
+ }
configuration_deepseek.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Deepseek Moe model configuration"""
2
+ from transformers.utils import logging
3
+ from transformers.configuration_utils import PretrainedConfig
4
+ from transformers.modeling_rope_utils import rope_config_validation
5
+
6
+ logger = logging.get_logger(__name__)
7
+
8
+ DEEPSEEK_FIXES_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
9
+ class DeepseekConfig(PretrainedConfig):
10
+ r"""
11
+ This is the configuration class to store the configuration of a DeepseekModel`]. It is used to instantiate an DeepSeek
12
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
13
+ defaults will yield a similar configuration to that of the DeepseekModel-20b.
14
+
15
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
16
+ documentation from [`PretrainedConfig`] for more information.
17
+
18
+
19
+ Args:
20
+ vocab_size (`int`, *optional*, defaults to 128256):
21
+ Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
22
+ `inputs_ids` passed when calling [`DeepseekModel`]
23
+ hidden_size (`int`, *optional*, defaults to 4096):
24
+ Dimension of the hidden representations.
25
+ intermediate_size (`int`, *optional*, defaults to 11008):
26
+ Dimension of the MLP representations.
27
+ moe_intermediate_size (`int`, *optional*, defaults to 1792):
28
+ Dimension of the MoE representations.
29
+ num_hidden_layers (`int`, *optional*, defaults to 32):
30
+ Number of hidden layers in the Transformer decoder.
31
+ num_attention_heads (`int`, *optional*, defaults to 32):
32
+ Number of attention heads for each attention layer in the Transformer decoder.
33
+ n_shared_experts (`int`, *optional*, defaults to None):
34
+ Number of shared experts, None means dense model.
35
+ n_routed_experts (`int`, *optional*, defaults to None):
36
+ Number of routed experts, None means dense model.
37
+ num_experts_per_tok (`int`, *optional*, defaults to None):
38
+ Number of selected experts, None means dense model.
39
+ moe_layer_freq (`int`, *optional*, defaults to 1):
40
+ The frequency of the MoE layer: one expert layer for every `moe_layer_freq - 1` dense layers.
41
+ first_k_dense_replace (`int`, *optional*, defaults to 0):
42
+ Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
43
+ \--k dense layers--/
44
+ norm_topk_prob (`bool`, *optional*, defaults to False):
45
+ Whether to normalize the weights of the routed experts.
46
+ scoring_func (`str`, *optional*, defaults to 'softmax'):
47
+ Method of computing expert weights.
48
+ aux_loss_alpha (`float`, *optional*, defaults to 0.001):
49
+ Auxiliary loss weight coefficient.
50
+ seq_aux = (`bool`, *optional*, defaults to True):
51
+ Whether to compute the auxiliary loss for each individual sample.
52
+ num_key_value_heads (`int`, *optional*):
53
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
54
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
55
+ `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
56
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
57
+ by meanpooling all the original heads within that group. For more details checkout [this
58
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
59
+ `num_attention_heads`.
60
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
61
+ The non-linear activation function (function or string) in the decoder.
62
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
63
+ The maximum sequence length that this model might ever be used with.
64
+ initializer_range (`float`, *optional*, defaults to 0.02):
65
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
66
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
67
+ The epsilon used by the rms normalization layers.
68
+ use_cache (`bool`, *optional*, defaults to `True`):
69
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
70
+ relevant if `config.is_decoder=True`.
71
+ pad_token_id (`int`, *optional*):
72
+ Padding token id.
73
+ bos_token_id (`int`, *optional*, defaults to 1):
74
+ Beginning of stream token id.
75
+ eos_token_id (`int`, *optional*, defaults to 2):
76
+ End of stream token id.
77
+ pretraining_tp (`int`, *optional*, defaults to 1):
78
+ Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
79
+ document](https://huggingface.co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism) to
80
+ understand more about it. This value is necessary to ensure exact reproducibility of the pretraining
81
+ results. Please refer to [this issue](https://github.com/pytorch/pytorch/issues/76232).
82
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
83
+ Whether to tie weight embeddings
84
+ rope_theta (`float`, *optional*, defaults to 10000.0):
85
+ The base period of the RoPE embeddings.
86
+ rope_scaling (`Dict`, *optional*):
87
+ Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
88
+ and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
89
+ accordingly.
90
+ Expected contents:
91
+ `rope_type` (`str`):
92
+ The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
93
+ 'llama3'], with 'default' being the original RoPE implementation.
94
+ `factor` (`float`, *optional*):
95
+ Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
96
+ most scaling types, a `factor` of x will enable the model to handle sequences of length x *
97
+ original maximum pre-trained length.
98
+ `original_max_position_embeddings` (`int`, *optional*):
99
+ Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
100
+ pretraining.
101
+ `attention_factor` (`float`, *optional*):
102
+ Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
103
+ computation. If unspecified, it defaults to value recommended by the implementation, using the
104
+ `factor` field to infer the suggested value.
105
+ `beta_fast` (`float`, *optional*):
106
+ Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
107
+ ramp function. If unspecified, it defaults to 32.
108
+ `beta_slow` (`float`, *optional*):
109
+ Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
110
+ ramp function. If unspecified, it defaults to 1.
111
+ `short_factor` (`List[float]`, *optional*):
112
+ Only used with 'longrope'. The scaling factor to be applied to short contexts (<
113
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
114
+ size divided by the number of attention heads divided by 2
115
+ `long_factor` (`List[float]`, *optional*):
116
+ Only used with 'longrope'. The scaling factor to be applied to long contexts (<
117
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
118
+ size divided by the number of attention heads divided by 2
119
+ `low_freq_factor` (`float`, *optional*):
120
+ Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
121
+ `high_freq_factor` (`float`, *optional*):
122
+ Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
123
+ attention_bias (`bool`, *optional*, defaults to `False`):
124
+ Whether to use a bias in the query, key, value and output projection layers during self-attention.
125
+ attention_dropout (`float`, *optional*, defaults to 0.0):
126
+ The dropout ratio for the attention probabilities.
127
+ mlp_bias (`bool`, *optional*, defaults to `False`):
128
+ Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.
129
+ head_dim (`int`, *optional*):
130
+ The attention head dimension. If None, it will default to hidden_size // num_heads
131
+
132
+ ```python
133
+ >>> from transformers import DeepseekModel, DeepseekConfig
134
+
135
+ >>> configuration = DeepseekConfig()
136
+ >>> model = DeepseekModel(configuration)
137
+
138
+ >>> # Accessing the model configuration
139
+ >>> configuration = model.config
140
+ ```"""
141
+
142
+ model_type = "deepseek"
143
+ keys_to_ignore_at_inference = ["past_key_values"]
144
+
145
+ def __init__(
146
+ self,
147
+ vocab_size=128256,
148
+ hidden_size=2048,
149
+ intermediate_size=14336,
150
+ moe_intermediate_size = 1792,
151
+ num_hidden_layers=28,
152
+ num_attention_heads=16,
153
+ num_key_value_heads=8,
154
+ n_shared_experts = None,
155
+ n_routed_experts = None,
156
+ num_experts_per_tok = None,
157
+ moe_layer_freq = 1,
158
+ first_k_dense_replace = 0,
159
+ norm_topk_prob = False,
160
+ scoring_func = 'softmax',
161
+ aux_loss_alpha = 0.001,
162
+ seq_aux = True,
163
+ hidden_act="silu",
164
+ max_position_embeddings=2048,
165
+ initializer_range=0.02,
166
+ rms_norm_eps=1e-6,
167
+ use_cache=True,
168
+ pad_token_id=None,
169
+ bos_token_id=1,
170
+ eos_token_id=2,
171
+ pretraining_tp=1,
172
+ tie_word_embeddings=False,
173
+ rope_theta=10000.0,
174
+ rope_scaling=None,
175
+ attention_bias=False,
176
+ attention_dropout=0.0,
177
+ moe_implementation="eager",
178
+ mlp_bias=False,
179
+ head_dim=None,
180
+ **kwargs,
181
+ ):
182
+ assert moe_implementation in ('eager', ), "Invalid moe_implementation value."
183
+ self.vocab_size = vocab_size
184
+ self.max_position_embeddings = max_position_embeddings
185
+ self.hidden_size = hidden_size
186
+ self.intermediate_size = intermediate_size
187
+ self.moe_intermediate_size = moe_intermediate_size
188
+ self.num_hidden_layers = num_hidden_layers
189
+ self.num_attention_heads = num_attention_heads
190
+ self.n_shared_experts = n_shared_experts
191
+ self.n_routed_experts = n_routed_experts
192
+ self.num_experts_per_tok = num_experts_per_tok
193
+ self.moe_layer_freq = moe_layer_freq
194
+ self.first_k_dense_replace = first_k_dense_replace
195
+ self.norm_topk_prob = norm_topk_prob
196
+ self.scoring_func = scoring_func
197
+ self.aux_loss_alpha = aux_loss_alpha
198
+ self.seq_aux = seq_aux
199
+
200
+ # for backward compatibility
201
+ if num_key_value_heads is None:
202
+ num_key_value_heads = num_attention_heads
203
+
204
+ self.num_key_value_heads = num_key_value_heads
205
+ self.hidden_act = hidden_act
206
+ self.initializer_range = initializer_range
207
+ self.rms_norm_eps = rms_norm_eps
208
+ self.pretraining_tp = pretraining_tp
209
+ self.use_cache = use_cache
210
+ self.rope_theta = rope_theta
211
+ self.rope_scaling = rope_scaling
212
+ self.attention_bias = attention_bias
213
+ self.attention_dropout = attention_dropout
214
+ self.mlp_bias = mlp_bias
215
+ self.head_dim = head_dim if head_dim is not None else self.hidden_size // self.num_attention_heads
216
+ # Validate the correctness of rotary position embeddings parameters
217
+ # BC: if there is a 'type' field, copy it it to 'rope_type'.
218
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
219
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
220
+ rope_config_validation(self)
221
+ self.moe_implementation = moe_implementation
222
+
223
+ super().__init__(
224
+ pad_token_id=pad_token_id,
225
+ bos_token_id=bos_token_id,
226
+ eos_token_id=eos_token_id,
227
+ tie_word_embeddings=tie_word_embeddings,
228
+ **kwargs,
229
+ )
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 2,
6
+ "transformers_version": "4.47.0"
7
+ }
model-00001-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7090612ca112fc336f35a3194868acd599b4b17dc555353dd69e942d1f39f0fb
3
+ size 4989712544
model-00002-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b6f8dfa2a4e9e82e950da5d7c9ab5cb0f94f46f74a334d5c89b07b412aa150e
3
+ size 4998095736
model-00003-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da64b3984a7c3ba2197e6257ff4d333fe8e065a5460816726d2e86839e6aea7c
3
+ size 4990247720
model-00004-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3d9ed394fcbdac4ae41986fbc7fbc671d3b511f7abb59c1bfa7170a40a53f5b
3
+ size 4990247720
model-00005-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:425361570c1abe64bb3640405ad68091229fa3f1f75ef290c70a305d3e1fcfab
3
+ size 4998095736
model-00006-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fae60f4453c0aab032815b5b0844b8e28891499aa40fe5b1df9bf40850ee0385
3
+ size 4990247856
model-00007-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d96696790e9d485ac865d30c7dde6f686298dc85a41bf406141b7446918449b8
3
+ size 4990248056
model-00008-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00c0aa73d325e6a1faad989925f92bce2c6bd2ac602a1b6f9c507fa56c86a2b1
3
+ size 4988150944
model-00009-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f8479a7ab064cbb74ba8391a015945e5d62dee8125d36a94d50bfda7f214dfed
3
+ size 4985513000
model-00010-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7dfdde6ded6e2daedd8ff8325a55937f29754720d98e115431742b9b1a6e151
3
+ size 4990248056
model-00011-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79ea477cd49ca8b94ac32c0a9c9400c25a84235bcfae10154fe148dbd283fb21
3
+ size 4990248080
model-00012-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0dd4c77c6713b30016d623979f228dda44027e9f190103f57a37834bcb31395
3
+ size 4998096056
model-00013-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ba04d3f2864f60e87ee0513f62f353bb085e506769ee58f0ea21639419e50b8
3
+ size 4990248056
model-00014-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e728bc7959e994881f618772a3f228c0b6409d4dc0e3b54e3a80725e7268427d
3
+ size 4990248064
model-00015-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d4843ff074ccca26eb0a2aa808cbf499df86805b546cef3e692173903644786
3
+ size 4998096072
model-00016-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8406276ffd313932dfa24128f8d94e957ba248aaa6b650a868b7f1e13b4db2ff
3
+ size 4990248056
model-00017-of-00017.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a559b6245dfb2e1b1156194f35f436a5bce2e440fe6a12cb1d1f05302d2fb082
3
+ size 2489880664
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
modelling_deepseek.py ADDED
@@ -0,0 +1,1413 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ """ PyTorch Deepseek Moe model with fixed Rope and updated code."""
3
+ import math
4
+ from typing import List, Optional, Tuple, Union
5
+
6
+ import torch
7
+ import torch.nn.functional as F
8
+ import torch.utils.checkpoint
9
+ from torch import nn
10
+
11
+ from transformers.activations import ACT2FN
12
+ from transformers.cache_utils import Cache, DynamicCache, StaticCache
13
+ from transformers.generation import GenerationMixin
14
+ from transformers.modeling_attn_mask_utils import AttentionMaskConverter
15
+ from transformers.modeling_flash_attention_utils import FlashAttentionKwargs, _flash_attention_forward
16
+ from transformers.modeling_outputs import (
17
+ BaseModelOutputWithPast,
18
+ CausalLMOutputWithPast,
19
+ SequenceClassifierOutputWithPast,
20
+ )
21
+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
22
+ from transformers.modeling_utils import PreTrainedModel
23
+ from transformers.processing_utils import Unpack
24
+ from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
25
+ from transformers.utils import (
26
+ LossKwargs,
27
+ add_start_docstrings,
28
+ add_start_docstrings_to_model_forward,
29
+ is_flash_attn_greater_or_equal_2_10,
30
+ logging,
31
+ replace_return_docstrings,
32
+ )
33
+ from .configuration_deepseek import DeepseekConfig
34
+
35
+
36
+ logger = logging.get_logger(__name__)
37
+
38
+ _CONFIG_FOR_DOC = "DeepseekConfig"
39
+
40
+
41
+ class DeepseekRMSNorm(nn.Module):
42
+ def __init__(self, hidden_size, eps=1e-6):
43
+ """
44
+ DeepseekRMSNorm is equivalent to T5LayerNorm
45
+ """
46
+ super().__init__()
47
+ self.weight = nn.Parameter(torch.ones(hidden_size))
48
+ self.variance_epsilon = eps
49
+
50
+ def forward(self, hidden_states):
51
+ input_dtype = hidden_states.dtype
52
+ hidden_states = hidden_states.to(torch.float32)
53
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
54
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
55
+ return self.weight * hidden_states.to(input_dtype)
56
+
57
+ def extra_repr(self):
58
+ return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
59
+
60
+
61
+ ALL_LAYERNORM_LAYERS.append(DeepseekRMSNorm)
62
+
63
+
64
+ class DeepseekRotaryEmbedding(nn.Module):
65
+ def __init__(
66
+ self,
67
+ dim=None,
68
+ max_position_embeddings=2048,
69
+ base=10000,
70
+ device=None,
71
+ scaling_factor=1.0,
72
+ rope_type="default",
73
+ config: Optional[DeepseekConfig] = None,
74
+ ):
75
+ super().__init__()
76
+ # TODO (joao): remove the `if` below, only used for BC
77
+ self.rope_kwargs = {}
78
+ if config is None:
79
+ logger.warning_once(
80
+ "`DeepseekRotaryEmbedding` can now be fully parameterized by passing the model config through the "
81
+ "`config` argument. All other arguments will be removed in v4.46"
82
+ )
83
+ self.rope_kwargs = {
84
+ "rope_type": rope_type,
85
+ "factor": scaling_factor,
86
+ "dim": dim,
87
+ "base": base,
88
+ "max_position_embeddings": max_position_embeddings,
89
+ }
90
+ self.rope_type = rope_type
91
+ self.max_seq_len_cached = max_position_embeddings
92
+ self.original_max_seq_len = max_position_embeddings
93
+ else:
94
+ # BC: "rope_type" was originally "type"
95
+ if config.rope_scaling is not None:
96
+ self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
97
+ else:
98
+ self.rope_type = "default"
99
+ self.max_seq_len_cached = config.max_position_embeddings
100
+ self.original_max_seq_len = config.max_position_embeddings
101
+
102
+ self.config = config
103
+ self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
104
+
105
+ inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, **self.rope_kwargs)
106
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
107
+ self.original_inv_freq = self.inv_freq
108
+
109
+ def _dynamic_frequency_update(self, position_ids, device):
110
+ """
111
+ dynamic RoPE layers should recompute `inv_freq` in the following situations:
112
+ 1 - growing beyond the cached sequence length (allow scaling)
113
+ 2 - the current sequence length is in the original scale (avoid losing precision with small sequences)
114
+ """
115
+ seq_len = torch.max(position_ids) + 1
116
+ if seq_len > self.max_seq_len_cached: # growth
117
+ inv_freq, self.attention_scaling = self.rope_init_fn(
118
+ self.config, device, seq_len=seq_len, **self.rope_kwargs
119
+ )
120
+ self.register_buffer("inv_freq", inv_freq, persistent=False) # TODO joao: may break with compilation
121
+ self.max_seq_len_cached = seq_len
122
+
123
+ if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len: # reset
124
+ self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
125
+ self.max_seq_len_cached = self.original_max_seq_len
126
+
127
+ @torch.no_grad()
128
+ def forward(self, x, position_ids):
129
+ if "dynamic" in self.rope_type:
130
+ self._dynamic_frequency_update(position_ids, device=x.device)
131
+
132
+ # Core RoPE block
133
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
134
+ position_ids_expanded = position_ids[:, None, :].float()
135
+ # Force float32 (see https://github.com/huggingface/transformers/pull/29285)
136
+ device_type = x.device.type
137
+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
138
+ with torch.autocast(device_type=device_type, enabled=False):
139
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
140
+ emb = torch.cat((freqs, freqs), dim=-1)
141
+ cos = emb.cos()
142
+ sin = emb.sin()
143
+
144
+ # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention
145
+ cos = cos * self.attention_scaling
146
+ sin = sin * self.attention_scaling
147
+
148
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
149
+
150
+
151
+ class DeepseekLinearScalingRotaryEmbedding(DeepseekRotaryEmbedding):
152
+ """DeepseekRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
153
+
154
+ def __init__(self, *args, **kwargs):
155
+ logger.warning_once(
156
+ "`DeepseekLinearScalingRotaryEmbedding` is deprecated an will be removed in v4.46. Please use "
157
+ "`DeepseekRotaryEmbedding`, which now also does linear scaling (simply pass the model config to __init__)."
158
+ )
159
+ kwargs["rope_type"] = "linear"
160
+ super().__init__(*args, **kwargs)
161
+
162
+
163
+ class DeepseekDynamicNTKScalingRotaryEmbedding(DeepseekRotaryEmbedding):
164
+ """DeepseekRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
165
+
166
+ def __init__(self, *args, **kwargs):
167
+ logger.warning_once(
168
+ "`DeepseekDynamicNTKScalingRotaryEmbedding` is deprecated an will be removed in v4.46. Please use "
169
+ "`DeepseekRotaryEmbedding`, which now also does dynamic ntk scaling (simply pass the model config to "
170
+ "__init__)."
171
+ )
172
+ kwargs["rope_type"] = "dynamic"
173
+ super().__init__(*args, **kwargs)
174
+
175
+
176
+ def rotate_half(x):
177
+ """Rotates half the hidden dims of the input."""
178
+ x1 = x[..., : x.shape[-1] // 2]
179
+ x2 = x[..., x.shape[-1] // 2 :]
180
+ return torch.cat((-x2, x1), dim=-1)
181
+
182
+
183
+ def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
184
+ """Applies Rotary Position Embedding to the query and key tensors.
185
+
186
+ Args:
187
+ q (`torch.Tensor`): The query tensor.
188
+ k (`torch.Tensor`): The key tensor.
189
+ cos (`torch.Tensor`): The cosine part of the rotary embedding.
190
+ sin (`torch.Tensor`): The sine part of the rotary embedding.
191
+ position_ids (`torch.Tensor`, *optional*):
192
+ Deprecated and unused.
193
+ unsqueeze_dim (`int`, *optional*, defaults to 1):
194
+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
195
+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
196
+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
197
+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
198
+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
199
+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
200
+ Returns:
201
+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
202
+ """
203
+ cos = cos.unsqueeze(unsqueeze_dim)
204
+ sin = sin.unsqueeze(unsqueeze_dim)
205
+ q_embed = (q * cos) + (rotate_half(q) * sin)
206
+ k_embed = (k * cos) + (rotate_half(k) * sin)
207
+ return q_embed, k_embed
208
+
209
+
210
+ class DeepseekMLP(nn.Module):
211
+ def __init__(self, config, hidden_size=None, intermediate_size=None):
212
+ super().__init__()
213
+ self.config = config
214
+ self.hidden_size = config.hidden_size if hidden_size is None else hidden_size
215
+ self.intermediate_size = config.intermediate_size if intermediate_size is None else intermediate_size
216
+
217
+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
218
+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
219
+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
220
+ self.act_fn = ACT2FN[config.hidden_act]
221
+
222
+ def forward(self, x, **kwargs):
223
+ down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
224
+ return down_proj
225
+
226
+
227
+ class MoEGate(nn.Module):
228
+ def __init__(self, config):
229
+ super().__init__()
230
+ self.config = config
231
+ self.top_k = config.num_experts_per_tok
232
+ self.n_routed_experts = config.n_routed_experts
233
+
234
+ self.scoring_func = config.scoring_func
235
+ self.alpha = config.aux_loss_alpha
236
+ self.seq_aux = config.seq_aux
237
+
238
+ # topk selection algorithm
239
+ self.norm_topk_prob = config.norm_topk_prob
240
+ self.gating_dim = config.hidden_size
241
+ self.weight = nn.Parameter(torch.empty((self.n_routed_experts, self.gating_dim)))
242
+
243
+ self.reset_parameters()
244
+
245
+ def reset_parameters(self) -> None:
246
+ import torch.nn.init as init
247
+ init.kaiming_uniform_(self.weight, a=math.sqrt(5))
248
+
249
+ def forward(self, hidden_states):
250
+ bsz, seq_len, h = hidden_states.shape
251
+ # Compute gating score
252
+ hidden_states = hidden_states.view(-1, h)
253
+ logits = F.linear(hidden_states, self.weight, None)
254
+ if self.scoring_func == 'softmax':
255
+ scores = logits.to(torch.float32).softmax(dim=-1)
256
+ else:
257
+ raise NotImplementedError(f'insupportable scoring function for MoE gating: {self.scoring_func}')
258
+
259
+ # Select top-k experts
260
+ topk_weight, topk_idx = torch.topk(scores, k=self.top_k, dim=-1, sorted=False)
261
+
262
+ # Norm gate to sum 1
263
+ if self.top_k > 1 and self.norm_topk_prob:
264
+ denominator = topk_weight.sum(dim=-1, keepdim=True) + 1e-20
265
+ topk_weight = topk_weight / denominator
266
+
267
+ # Expert-level computation auxiliary loss
268
+ aux_loss = None
269
+ return topk_idx, topk_weight.to(hidden_states.dtype), aux_loss
270
+
271
+
272
+ class AddAuxiliaryLoss(torch.autograd.Function):
273
+ """
274
+ The trick function of adding auxiliary (aux) loss,
275
+ which includes the gradient of the aux loss during backpropagation.
276
+ """
277
+
278
+ @staticmethod
279
+ def forward(ctx, x, loss):
280
+ assert loss.numel() == 1
281
+ ctx.dtype = loss.dtype
282
+ ctx.required_aux_loss = loss.requires_grad
283
+ return x
284
+
285
+ @staticmethod
286
+ def backward(ctx, grad_output):
287
+ grad_loss = None
288
+ if ctx.required_aux_loss:
289
+ grad_loss = torch.ones(1, dtype=ctx.dtype, device=grad_output.device)
290
+ return grad_output, grad_loss
291
+
292
+
293
+ class DeepseekMoE(nn.Module):
294
+ """
295
+ A mixed expert module containing shared experts.
296
+ """
297
+
298
+ def __init__(self, config):
299
+ super().__init__()
300
+ self.config = config
301
+ self.num_experts_per_tok = config.num_experts_per_tok
302
+ self.experts = nn.ModuleList(
303
+ [DeepseekMLP(config, intermediate_size=config.moe_intermediate_size) for i in
304
+ range(config.n_routed_experts)])
305
+ self.gate = MoEGate(config)
306
+ if config.n_shared_experts is not None:
307
+ intermediate_size = config.moe_intermediate_size * config.n_shared_experts
308
+ self.shared_experts = DeepseekMLP(config=config, intermediate_size=intermediate_size)
309
+
310
+ def forward(self, hidden_states):
311
+ identity = hidden_states
312
+ orig_shape = hidden_states.shape
313
+ topk_idx, topk_weight, aux_loss = self.gate(hidden_states)
314
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
315
+ flat_topk_idx = topk_idx.view(-1)
316
+ if self.training:
317
+ y = self.moe_train(hidden_states, flat_topk_idx, topk_weight.view(-1, 1))
318
+ y = y.view(*orig_shape)
319
+ y = AddAuxiliaryLoss.apply(y, aux_loss)
320
+ else:
321
+ y = self.moe_infer(hidden_states, flat_topk_idx, topk_weight.view(-1, 1)).view(*orig_shape)
322
+ if self.config.n_shared_experts is not None:
323
+ y = y + self.shared_experts(identity)
324
+ return y
325
+
326
+ def moe_train(self, hidden_states, flat_topk_idx, topk_weight):
327
+ hidden_states = hidden_states.repeat_interleave(self.num_experts_per_tok, dim=0)
328
+ y = torch.empty_like(hidden_states)
329
+ for i, expert in enumerate(self.experts):
330
+ y[flat_topk_idx == i] = expert(hidden_states[flat_topk_idx == i])
331
+ y = (y.view(*topk_weight.shape, -1) * topk_weight.unsqueeze(-1)).sum(dim=1)
332
+ return y
333
+
334
+ @torch.no_grad()
335
+ def moe_infer(self, x, flat_expert_indices, flat_expert_weights):
336
+ expert_cache = torch.zeros_like(x)
337
+ idxs = flat_expert_indices.argsort()
338
+ tokens_per_expert = flat_expert_indices.bincount().cpu().numpy().cumsum(0)
339
+ token_idxs = idxs // self.num_experts_per_tok
340
+ for i, end_idx in enumerate(tokens_per_expert):
341
+ start_idx = 0 if i == 0 else tokens_per_expert[i - 1]
342
+ if start_idx == end_idx:
343
+ continue
344
+ expert = self.experts[i]
345
+ exp_token_idx = token_idxs[start_idx:end_idx]
346
+ expert_tokens = x[exp_token_idx]
347
+ expert_out = expert(expert_tokens)
348
+ expert_out.mul_(flat_expert_weights[idxs[start_idx:end_idx]])
349
+ expert_cache.scatter_reduce_(0, exp_token_idx.view(-1, 1).repeat(1, x.shape[-1]), expert_out, reduce='sum')
350
+ return expert_cache
351
+
352
+
353
+ Deepseek_MOE_CLASSES = {
354
+ 'eager': DeepseekMoE,
355
+ }
356
+
357
+ # Copied from transformers.models.llama.modeling_llama.repeat_kv
358
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
359
+ """
360
+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
361
+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
362
+ """
363
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
364
+ if n_rep == 1:
365
+ return hidden_states
366
+ hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
367
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
368
+
369
+
370
+ # Copied from transformers.models.llama.modeling_llama.LlamaAttention with Llama->Deepseek
371
+ class DeepseekAttention(nn.Module):
372
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
373
+
374
+ def __init__(self, config: DeepseekConfig, layer_idx: Optional[int] = None):
375
+ super().__init__()
376
+ self.config = config
377
+ self.layer_idx = layer_idx
378
+ if layer_idx is None:
379
+ logger.warning_once(
380
+ f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will "
381
+ "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` "
382
+ "when creating this class."
383
+ )
384
+
385
+ self.attention_dropout = config.attention_dropout
386
+ self.hidden_size = config.hidden_size
387
+ self.num_heads = config.num_attention_heads
388
+ self.head_dim = getattr(config, "head_dim", self.hidden_size // self.num_heads)
389
+ self.num_key_value_heads = config.num_key_value_heads
390
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
391
+ self.max_position_embeddings = config.max_position_embeddings
392
+ self.rope_theta = config.rope_theta
393
+ self.is_causal = True
394
+
395
+ self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias)
396
+ self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
397
+ self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
398
+ self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.attention_bias)
399
+
400
+ # TODO (joao): remove in v4.46 (RoPE is computed in the model, not in the decoder layers)
401
+ self.rotary_emb = DeepseekRotaryEmbedding(config=self.config)
402
+
403
+ def forward(
404
+ self,
405
+ hidden_states: torch.Tensor,
406
+ attention_mask: Optional[torch.Tensor] = None,
407
+ position_ids: Optional[torch.LongTensor] = None,
408
+ past_key_value: Optional[Cache] = None,
409
+ output_attentions: bool = False,
410
+ use_cache: bool = False,
411
+ cache_position: Optional[torch.LongTensor] = None,
412
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46
413
+ **kwargs,
414
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
415
+ bsz, q_len, _ = hidden_states.size()
416
+
417
+ query_states = self.q_proj(hidden_states)
418
+ key_states = self.k_proj(hidden_states)
419
+ value_states = self.v_proj(hidden_states)
420
+
421
+ # use -1 to infer num_heads and num_key_value_heads as they may vary if tensor parallel is used
422
+ query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
423
+ key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
424
+ value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
425
+
426
+ if position_embeddings is None:
427
+ logger.warning_once(
428
+ "The attention layers in this model are transitioning from computing the RoPE embeddings internally "
429
+ "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed "
430
+ "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be "
431
+ "removed and `position_embeddings` will be mandatory."
432
+ )
433
+ cos, sin = self.rotary_emb(value_states, position_ids)
434
+ else:
435
+ cos, sin = position_embeddings
436
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
437
+
438
+ if past_key_value is not None:
439
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
440
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
441
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
442
+
443
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
444
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
445
+ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
446
+
447
+ if attention_mask is not None: # no matter the length, we just slice it
448
+ causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
449
+ attn_weights = attn_weights + causal_mask
450
+
451
+ # upcast attention to fp32
452
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
453
+ attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
454
+ attn_output = torch.matmul(attn_weights, value_states)
455
+
456
+ if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
457
+ raise ValueError(
458
+ f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
459
+ f" {attn_output.size()}"
460
+ )
461
+
462
+ attn_output = attn_output.transpose(1, 2).contiguous()
463
+
464
+ attn_output = attn_output.reshape(bsz, q_len, -1)
465
+
466
+ attn_output = self.o_proj(attn_output)
467
+
468
+ if not output_attentions:
469
+ attn_weights = None
470
+
471
+ return attn_output, attn_weights, past_key_value
472
+
473
+
474
+ # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2 with Llama->Deepseek
475
+ class DeepseekFlashAttention2(DeepseekAttention):
476
+ """
477
+ Deepseek flash attention module. This module inherits from `DeepseekAttention` as the weights of the module stays
478
+ untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
479
+ flash attention and deal with padding tokens in case the input contains any of them.
480
+ """
481
+
482
+ def __init__(self, *args, **kwargs):
483
+ super().__init__(*args, **kwargs)
484
+
485
+ # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
486
+ # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
487
+ # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
488
+ self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
489
+
490
+ def forward(
491
+ self,
492
+ hidden_states: torch.Tensor,
493
+ attention_mask: Optional[torch.LongTensor] = None,
494
+ position_ids: Optional[torch.LongTensor] = None,
495
+ past_key_value: Optional[Cache] = None,
496
+ output_attentions: bool = False,
497
+ use_cache: bool = False,
498
+ cache_position: Optional[torch.LongTensor] = None,
499
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46
500
+ **kwargs: Unpack[FlashAttentionKwargs],
501
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
502
+ if isinstance(past_key_value, StaticCache):
503
+ raise ValueError(
504
+ "`static` cache implementation is not compatible with `attn_implementation==flash_attention_2` "
505
+ "make sure to use `sdpa` in the mean time, and open an issue at https://github.com/huggingface/transformers"
506
+ )
507
+
508
+ output_attentions = False
509
+
510
+ bsz, q_len, _ = hidden_states.size()
511
+
512
+ query_states = self.q_proj(hidden_states)
513
+ key_states = self.k_proj(hidden_states)
514
+ value_states = self.v_proj(hidden_states)
515
+
516
+ # Flash attention requires the input to have the shape
517
+ # batch_size x seq_length x head_dim x hidden_dim
518
+ # therefore we just need to keep the original shape
519
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
520
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
521
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
522
+
523
+ if position_embeddings is None:
524
+ logger.warning_once(
525
+ "The attention layers in this model are transitioning from computing the RoPE embeddings internally "
526
+ "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed "
527
+ "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be "
528
+ "removed and `position_embeddings` will be mandatory."
529
+ )
530
+ cos, sin = self.rotary_emb(value_states, position_ids)
531
+ else:
532
+ cos, sin = position_embeddings
533
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
534
+
535
+ if past_key_value is not None:
536
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
537
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
538
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
539
+
540
+ # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
541
+ # to be able to avoid many of these transpose/reshape/view.
542
+ query_states = query_states.transpose(1, 2)
543
+ key_states = key_states.transpose(1, 2)
544
+ value_states = value_states.transpose(1, 2)
545
+
546
+ dropout_rate = self.attention_dropout if self.training else 0.0
547
+
548
+ # In PEFT, usually we cast the layer norms in float32 for training stability reasons
549
+ # therefore the input hidden states gets silently casted in float32. Hence, we need
550
+ # cast them back in the correct dtype just to be sure everything works as expected.
551
+ # This might slowdown training & inference so it is recommended to not cast the LayerNorms
552
+ # in fp32. (DeepseekRMSNorm handles it correctly)
553
+
554
+ input_dtype = query_states.dtype
555
+ if input_dtype == torch.float32:
556
+ if torch.is_autocast_enabled():
557
+ target_dtype = torch.get_autocast_gpu_dtype()
558
+ # Handle the case where the model is quantized
559
+ elif hasattr(self.config, "_pre_quantization_dtype"):
560
+ target_dtype = self.config._pre_quantization_dtype
561
+ else:
562
+ target_dtype = self.q_proj.weight.dtype
563
+
564
+ logger.warning_once(
565
+ f"The input hidden states seems to be silently casted in float32, this might be related to"
566
+ f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
567
+ f" {target_dtype}."
568
+ )
569
+
570
+ query_states = query_states.to(target_dtype)
571
+ key_states = key_states.to(target_dtype)
572
+ value_states = value_states.to(target_dtype)
573
+
574
+ attn_output = _flash_attention_forward(
575
+ query_states,
576
+ key_states,
577
+ value_states,
578
+ attention_mask,
579
+ q_len,
580
+ position_ids=position_ids,
581
+ dropout=dropout_rate,
582
+ sliding_window=getattr(self, "sliding_window", None),
583
+ use_top_left_mask=self._flash_attn_uses_top_left_mask,
584
+ is_causal=self.is_causal,
585
+ **kwargs,
586
+ )
587
+
588
+ attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
589
+ attn_output = self.o_proj(attn_output)
590
+
591
+ if not output_attentions:
592
+ attn_weights = None
593
+
594
+ return attn_output, attn_weights, past_key_value
595
+
596
+ # Copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Deepseek
597
+ class DeepseekSdpaAttention(DeepseekAttention):
598
+ """
599
+ Deepseek attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
600
+ `DeepseekAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
601
+ SDPA API.
602
+ """
603
+
604
+ # Adapted from DeepseekAttention.forward
605
+ def forward(
606
+ self,
607
+ hidden_states: torch.Tensor,
608
+ attention_mask: Optional[torch.Tensor] = None,
609
+ position_ids: Optional[torch.LongTensor] = None,
610
+ past_key_value: Optional[Cache] = None,
611
+ output_attentions: bool = False,
612
+ use_cache: bool = False,
613
+ cache_position: Optional[torch.LongTensor] = None,
614
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46
615
+ **kwargs,
616
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
617
+ if output_attentions:
618
+ # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
619
+ logger.warning_once(
620
+ "DeepseekModel is using DeepseekSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
621
+ 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
622
+ )
623
+ return super().forward(
624
+ hidden_states=hidden_states,
625
+ attention_mask=attention_mask,
626
+ position_ids=position_ids,
627
+ past_key_value=past_key_value,
628
+ output_attentions=output_attentions,
629
+ use_cache=use_cache,
630
+ cache_position=cache_position,
631
+ position_embeddings=position_embeddings,
632
+ )
633
+
634
+ bsz, q_len, _ = hidden_states.size()
635
+
636
+ query_states = self.q_proj(hidden_states)
637
+ key_states = self.k_proj(hidden_states)
638
+ value_states = self.v_proj(hidden_states)
639
+
640
+ # use -1 to infer num_heads and num_key_value_heads as they may vary if tensor parallel is used
641
+ query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
642
+ key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
643
+ value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
644
+
645
+ if position_embeddings is None:
646
+ logger.warning_once(
647
+ "The attention layers in this model are transitioning from computing the RoPE embeddings internally "
648
+ "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed "
649
+ "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be "
650
+ "removed and `position_embeddings` will be mandatory."
651
+ )
652
+ cos, sin = self.rotary_emb(value_states, position_ids)
653
+ else:
654
+ cos, sin = position_embeddings
655
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
656
+
657
+ if past_key_value is not None:
658
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
659
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
660
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
661
+
662
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
663
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
664
+
665
+ causal_mask = attention_mask
666
+ if attention_mask is not None:
667
+ causal_mask = causal_mask[:, :, :, : key_states.shape[-2]]
668
+
669
+ # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
670
+ # Reference: https://github.com/pytorch/pytorch/issues/112577.
671
+ if query_states.device.type == "cuda" and causal_mask is not None:
672
+ query_states = query_states.contiguous()
673
+ key_states = key_states.contiguous()
674
+ value_states = value_states.contiguous()
675
+
676
+ # We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment
677
+ # in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling.
678
+ is_causal = True if causal_mask is None and q_len > 1 else False
679
+
680
+ attn_output = torch.nn.functional.scaled_dot_product_attention(
681
+ query_states,
682
+ key_states,
683
+ value_states,
684
+ attn_mask=causal_mask,
685
+ dropout_p=self.attention_dropout if self.training else 0.0,
686
+ is_causal=is_causal,
687
+ )
688
+
689
+ attn_output = attn_output.transpose(1, 2).contiguous()
690
+ attn_output = attn_output.view(bsz, q_len, -1)
691
+
692
+ attn_output = self.o_proj(attn_output)
693
+
694
+ return attn_output, None, past_key_value
695
+
696
+
697
+ Deepseek_ATTENTION_CLASSES = {
698
+ "eager": DeepseekAttention,
699
+ "flash_attention_2": DeepseekFlashAttention2,
700
+ "sdpa": DeepseekSdpaAttention,
701
+ }
702
+
703
+
704
+ class DeepseekDecoderLayer(nn.Module):
705
+ def __init__(self, config: DeepseekConfig, layer_idx: int):
706
+ super().__init__()
707
+ self.hidden_size = config.hidden_size
708
+
709
+ self.self_attn = Deepseek_ATTENTION_CLASSES[config._attn_implementation](config=config,
710
+ layer_idx=layer_idx)
711
+
712
+ self.mlp = Deepseek_MOE_CLASSES[config.moe_implementation](config) if (config.n_routed_experts is not None and \
713
+ layer_idx >= config.first_k_dense_replace and layer_idx % config.moe_layer_freq == 0) \
714
+ else DeepseekMLP(config)
715
+ self.input_layernorm = DeepseekRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
716
+ self.post_attention_layernorm = DeepseekRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
717
+
718
+ def forward(
719
+ self,
720
+ hidden_states: torch.Tensor,
721
+ attention_mask: Optional[torch.Tensor] = None,
722
+ position_ids: Optional[torch.LongTensor] = None,
723
+ past_key_value: Optional[Cache] = None,
724
+ output_attentions: Optional[bool] = False,
725
+ use_cache: Optional[bool] = False,
726
+ cache_position: Optional[torch.LongTensor] = None,
727
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46
728
+ **kwargs,
729
+ ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
730
+ """
731
+ Args:
732
+ hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
733
+ attention_mask (`torch.FloatTensor`, *optional*):
734
+ attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
735
+ query_sequence_length, key_sequence_length)` if default attention is used.
736
+ output_attentions (`bool`, *optional*):
737
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
738
+ returned tensors for more detail.
739
+ use_cache (`bool`, *optional*):
740
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
741
+ (see `past_key_values`).
742
+ past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
743
+ cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
744
+ Indices depicting the position of the input sequence tokens in the sequence
745
+ position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
746
+ Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
747
+ with `head_dim` being the embedding dimension of each attention head.
748
+ kwargs (`dict`, *optional*):
749
+ Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
750
+ into the model
751
+ """
752
+ residual = hidden_states
753
+
754
+ hidden_states = self.input_layernorm(hidden_states)
755
+
756
+ # Self Attention
757
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
758
+ hidden_states=hidden_states,
759
+ attention_mask=attention_mask,
760
+ position_ids=position_ids,
761
+ past_key_value=past_key_value,
762
+ output_attentions=output_attentions,
763
+ use_cache=use_cache,
764
+ cache_position=cache_position,
765
+ position_embeddings=position_embeddings,
766
+ **kwargs,
767
+ )
768
+ hidden_states = residual + hidden_states
769
+
770
+ # Fully Connected
771
+ residual = hidden_states
772
+ hidden_states = self.post_attention_layernorm(hidden_states)
773
+ hidden_states = self.mlp(hidden_states)
774
+ hidden_states = residual + hidden_states
775
+
776
+ outputs = (hidden_states,)
777
+
778
+ if output_attentions:
779
+ outputs += (self_attn_weights,)
780
+
781
+ if use_cache:
782
+ outputs += (present_key_value,)
783
+
784
+ return outputs
785
+
786
+
787
+ Deepseek_START_DOCSTRING = r"""
788
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
789
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
790
+ etc.)
791
+
792
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
793
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
794
+ and behavior.
795
+
796
+ Parameters:
797
+ config ([`DeepseekConfig`]):
798
+ Model configuration class with all the parameters of the model. Initializing with a config file does not
799
+ load the weights associated with the model, only the configuration. Check out the
800
+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.
801
+ """
802
+
803
+
804
+ @add_start_docstrings(
805
+ "The bare Deepseek Model outputting raw hidden-states without any specific head on top.",
806
+ Deepseek_START_DOCSTRING,
807
+ )
808
+ class DeepseekPreTrainedModel(PreTrainedModel):
809
+ config_class = DeepseekConfig
810
+ base_model_prefix = "model"
811
+ supports_gradient_checkpointing = True
812
+ _no_split_modules = ["DeepseekDecoderLayer"]
813
+ _skip_keys_device_placement = "past_key_values"
814
+ _supports_flash_attn_2 = True
815
+ _supports_sdpa = True
816
+ _supports_cache_class = True
817
+ _supports_quantized_cache = True
818
+ _supports_static_cache = True
819
+
820
+ def _init_weights(self, module):
821
+ std = self.config.initializer_range
822
+ if isinstance(module, nn.Linear):
823
+ module.weight.data.normal_(mean=0.0, std=std)
824
+ if module.bias is not None:
825
+ module.bias.data.zero_()
826
+ elif isinstance(module, nn.Embedding):
827
+ module.weight.data.normal_(mean=0.0, std=std)
828
+ if module.padding_idx is not None:
829
+ module.weight.data[module.padding_idx].zero_()
830
+
831
+
832
+ Deepseek_INPUTS_DOCSTRING = r"""
833
+ Args:
834
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
835
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
836
+ it.
837
+
838
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
839
+ [`PreTrainedTokenizer.__call__`] for details.
840
+
841
+ [What are input IDs?](../glossary#input-ids)
842
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
843
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
844
+
845
+ - 1 for tokens that are **not masked**,
846
+ - 0 for tokens that are **masked**.
847
+
848
+ [What are attention masks?](../glossary#attention-mask)
849
+
850
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
851
+ [`PreTrainedTokenizer.__call__`] for details.
852
+
853
+ If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
854
+ `past_key_values`).
855
+
856
+ If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
857
+ and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
858
+ information on the default strategy.
859
+
860
+ - 1 indicates the head is **not masked**,
861
+ - 0 indicates the head is **masked**.
862
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
863
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
864
+ config.n_positions - 1]`.
865
+
866
+ [What are position IDs?](../glossary#position-ids)
867
+ past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
868
+ Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
869
+ blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
870
+ returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
871
+
872
+ Two formats are allowed:
873
+ - a [`~cache_utils.Cache`] instance, see our
874
+ [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);
875
+ - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
876
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
877
+ cache format.
878
+
879
+ The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
880
+ legacy cache format will be returned.
881
+
882
+ If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
883
+ have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
884
+ of shape `(batch_size, sequence_length)`.
885
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
886
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
887
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
888
+ model's internal embedding lookup matrix.
889
+ use_cache (`bool`, *optional*):
890
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
891
+ `past_key_values`).
892
+ output_attentions (`bool`, *optional*):
893
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
894
+ tensors for more detail.
895
+ output_hidden_states (`bool`, *optional*):
896
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
897
+ more detail.
898
+ return_dict (`bool`, *optional*):
899
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
900
+ cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
901
+ Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
902
+ this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
903
+ the complete sequence length.
904
+ """
905
+
906
+
907
+ @add_start_docstrings(
908
+ "The bare Deepseek Model outputting raw hidden-states without any specific head on top.",
909
+ Deepseek_START_DOCSTRING,
910
+ )
911
+ class DeepseekModel(DeepseekPreTrainedModel):
912
+ """
913
+ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`DeepseekDecoderLayer`]
914
+
915
+ Args:
916
+ config: DeepseekConfig
917
+ """
918
+
919
+ def __init__(self, config: DeepseekConfig):
920
+ super().__init__(config)
921
+ self.padding_idx = config.pad_token_id
922
+ self.vocab_size = config.vocab_size
923
+
924
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
925
+ self.layers = nn.ModuleList(
926
+ [DeepseekDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
927
+ )
928
+ self.norm = DeepseekRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
929
+ self.rotary_emb = DeepseekRotaryEmbedding(config=config)
930
+
931
+ self.gradient_checkpointing = False
932
+ if getattr(config, "pretraining_tp", 1) != 1:
933
+ logger.warn("`pretraining_tp` is deprecated, please use `model.tensor_parallel` instead.")
934
+
935
+ # Initialize weights and apply final processing
936
+ self.post_init()
937
+
938
+ def get_input_embeddings(self):
939
+ return self.embed_tokens
940
+
941
+ def set_input_embeddings(self, value):
942
+ self.embed_tokens = value
943
+
944
+ @add_start_docstrings_to_model_forward(Deepseek_INPUTS_DOCSTRING)
945
+ def forward(
946
+ self,
947
+ input_ids: torch.LongTensor = None,
948
+ attention_mask: Optional[torch.Tensor] = None,
949
+ position_ids: Optional[torch.LongTensor] = None,
950
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
951
+ inputs_embeds: Optional[torch.FloatTensor] = None,
952
+ use_cache: Optional[bool] = None,
953
+ output_attentions: Optional[bool] = None,
954
+ output_hidden_states: Optional[bool] = None,
955
+ return_dict: Optional[bool] = None,
956
+ cache_position: Optional[torch.LongTensor] = None,
957
+ **flash_attn_kwargs: Unpack[FlashAttentionKwargs],
958
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
959
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
960
+ output_hidden_states = (
961
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
962
+ )
963
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
964
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
965
+
966
+ if (input_ids is None) ^ (inputs_embeds is not None):
967
+ raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
968
+
969
+ if self.gradient_checkpointing and self.training and use_cache:
970
+ logger.warning_once(
971
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
972
+ )
973
+ use_cache = False
974
+
975
+ if inputs_embeds is None:
976
+ inputs_embeds = self.embed_tokens(input_ids)
977
+
978
+ # kept for BC (non `Cache` `past_key_values` inputs)
979
+ return_legacy_cache = False
980
+ if use_cache and not isinstance(past_key_values, Cache):
981
+ return_legacy_cache = True
982
+ if past_key_values is None:
983
+ past_key_values = DynamicCache()
984
+ else:
985
+ past_key_values = DynamicCache.from_legacy_cache(past_key_values)
986
+ logger.warning_once(
987
+ "We detected that you are passing `past_key_values` as a tuple of tuples. This is deprecated and "
988
+ "will be removed in v4.47. Please convert your cache or use an appropriate `Cache` class "
989
+ "(https://huggingface.co/docs/transformers/kv_cache#legacy-cache-format)"
990
+ )
991
+
992
+ if cache_position is None:
993
+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
994
+ cache_position = torch.arange(
995
+ past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
996
+ )
997
+ if position_ids is None:
998
+ position_ids = cache_position.unsqueeze(0)
999
+
1000
+ causal_mask = self._update_causal_mask(
1001
+ attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions
1002
+ )
1003
+ hidden_states = inputs_embeds
1004
+
1005
+ # create position embeddings to be shared across the decoder layers
1006
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
1007
+
1008
+ # decoder layers
1009
+ all_hidden_states = () if output_hidden_states else None
1010
+ all_self_attns = () if output_attentions else None
1011
+ next_decoder_cache = None
1012
+
1013
+ for decoder_layer in self.layers[: self.config.num_hidden_layers]:
1014
+ if output_hidden_states:
1015
+ all_hidden_states += (hidden_states,)
1016
+
1017
+ if self.gradient_checkpointing and self.training:
1018
+ layer_outputs = self._gradient_checkpointing_func(
1019
+ decoder_layer.__call__,
1020
+ hidden_states,
1021
+ causal_mask,
1022
+ position_ids,
1023
+ past_key_values,
1024
+ output_attentions,
1025
+ use_cache,
1026
+ cache_position,
1027
+ position_embeddings,
1028
+ )
1029
+ else:
1030
+ layer_outputs = decoder_layer(
1031
+ hidden_states,
1032
+ attention_mask=causal_mask,
1033
+ position_ids=position_ids,
1034
+ past_key_value=past_key_values,
1035
+ output_attentions=output_attentions,
1036
+ use_cache=use_cache,
1037
+ cache_position=cache_position,
1038
+ position_embeddings=position_embeddings,
1039
+ **flash_attn_kwargs,
1040
+ )
1041
+
1042
+ hidden_states = layer_outputs[0]
1043
+
1044
+ if use_cache:
1045
+ next_decoder_cache = layer_outputs[2 if output_attentions else 1]
1046
+
1047
+ if output_attentions:
1048
+ all_self_attns += (layer_outputs[1],)
1049
+
1050
+ hidden_states = self.norm(hidden_states)
1051
+
1052
+ # add hidden states from the last decoder layer
1053
+ if output_hidden_states:
1054
+ all_hidden_states += (hidden_states,)
1055
+
1056
+ next_cache = next_decoder_cache if use_cache else None
1057
+ if return_legacy_cache:
1058
+ next_cache = next_cache.to_legacy_cache()
1059
+
1060
+ if not return_dict:
1061
+ return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
1062
+ return BaseModelOutputWithPast(
1063
+ last_hidden_state=hidden_states,
1064
+ past_key_values=next_cache,
1065
+ hidden_states=all_hidden_states,
1066
+ attentions=all_self_attns,
1067
+ )
1068
+
1069
+ def _update_causal_mask(
1070
+ self,
1071
+ attention_mask: torch.Tensor,
1072
+ input_tensor: torch.Tensor,
1073
+ cache_position: torch.Tensor,
1074
+ past_key_values: Cache,
1075
+ output_attentions: bool,
1076
+ ):
1077
+ if self.config._attn_implementation == "flash_attention_2":
1078
+ if attention_mask is not None and 0.0 in attention_mask:
1079
+ return attention_mask
1080
+ return None
1081
+
1082
+ # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
1083
+ # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
1084
+ # to infer the attention mask.
1085
+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
1086
+ using_static_cache = isinstance(past_key_values, StaticCache)
1087
+
1088
+ # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
1089
+ if self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions:
1090
+ if AttentionMaskConverter._ignore_causal_mask_sdpa(
1091
+ attention_mask,
1092
+ inputs_embeds=input_tensor,
1093
+ past_key_values_length=past_seen_tokens,
1094
+ is_training=self.training,
1095
+ ):
1096
+ return None
1097
+
1098
+ dtype, device = input_tensor.dtype, input_tensor.device
1099
+ sequence_length = input_tensor.shape[1]
1100
+ if using_static_cache:
1101
+ target_length = past_key_values.get_max_cache_shape()
1102
+ else:
1103
+ target_length = (
1104
+ attention_mask.shape[-1]
1105
+ if isinstance(attention_mask, torch.Tensor)
1106
+ else past_seen_tokens + sequence_length + 1
1107
+ )
1108
+
1109
+ # In case the provided `attention` mask is 2D, we generate a causal mask here (4D).
1110
+ causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
1111
+ attention_mask,
1112
+ sequence_length=sequence_length,
1113
+ target_length=target_length,
1114
+ dtype=dtype,
1115
+ device=device,
1116
+ cache_position=cache_position,
1117
+ batch_size=input_tensor.shape[0],
1118
+ )
1119
+
1120
+ if (
1121
+ self.config._attn_implementation == "sdpa"
1122
+ and attention_mask is not None
1123
+ and attention_mask.device.type == "cuda"
1124
+ and not output_attentions
1125
+ ):
1126
+ # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
1127
+ # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
1128
+ # Details: https://github.com/pytorch/pytorch/issues/110213
1129
+ min_dtype = torch.finfo(dtype).min
1130
+ causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype)
1131
+
1132
+ return causal_mask
1133
+
1134
+ @staticmethod
1135
+ def _prepare_4d_causal_attention_mask_with_cache_position(
1136
+ attention_mask: torch.Tensor,
1137
+ sequence_length: int,
1138
+ target_length: int,
1139
+ dtype: torch.dtype,
1140
+ device: torch.device,
1141
+ cache_position: torch.Tensor,
1142
+ batch_size: int,
1143
+ **kwargs,
1144
+ ):
1145
+ """
1146
+ Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
1147
+ `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
1148
+
1149
+ Args:
1150
+ attention_mask (`torch.Tensor`):
1151
+ A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
1152
+ `(batch_size, 1, query_length, key_value_length)`.
1153
+ sequence_length (`int`):
1154
+ The sequence length being processed.
1155
+ target_length (`int`):
1156
+ The target length: when generating with static cache, the mask should be as long as the static cache,
1157
+ to account for the 0 padding, the part of the cache that is not filled yet.
1158
+ dtype (`torch.dtype`):
1159
+ The dtype to use for the 4D attention mask.
1160
+ device (`torch.device`):
1161
+ The device to plcae the 4D attention mask on.
1162
+ cache_position (`torch.Tensor`):
1163
+ Indices depicting the position of the input sequence tokens in the sequence.
1164
+ batch_size (`torch.Tensor`):
1165
+ Batch size.
1166
+ """
1167
+ if attention_mask is not None and attention_mask.dim() == 4:
1168
+ # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
1169
+ causal_mask = attention_mask
1170
+ else:
1171
+ min_dtype = torch.finfo(dtype).min
1172
+ causal_mask = torch.full(
1173
+ (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device
1174
+ )
1175
+ if sequence_length != 1:
1176
+ causal_mask = torch.triu(causal_mask, diagonal=1)
1177
+ causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1)
1178
+ causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
1179
+ if attention_mask is not None:
1180
+ causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit
1181
+ mask_length = attention_mask.shape[-1]
1182
+ padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :]
1183
+ padding_mask = padding_mask == 0
1184
+ causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(
1185
+ padding_mask, min_dtype
1186
+ )
1187
+
1188
+ return causal_mask
1189
+
1190
+
1191
+ class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ...
1192
+
1193
+
1194
+ class DeepseekForCausalLM(DeepseekPreTrainedModel, GenerationMixin):
1195
+ _tied_weights_keys = ["lm_head.weight"]
1196
+
1197
+ def __init__(self, config):
1198
+ super().__init__(config)
1199
+ self.model = DeepseekModel(config)
1200
+ self.vocab_size = config.vocab_size
1201
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
1202
+
1203
+ # Initialize weights and apply final processing
1204
+ self.post_init()
1205
+
1206
+ def get_input_embeddings(self):
1207
+ return self.model.embed_tokens
1208
+
1209
+ def set_input_embeddings(self, value):
1210
+ self.model.embed_tokens = value
1211
+
1212
+ def get_output_embeddings(self):
1213
+ return self.lm_head
1214
+
1215
+ def set_output_embeddings(self, new_embeddings):
1216
+ self.lm_head = new_embeddings
1217
+
1218
+ def set_decoder(self, decoder):
1219
+ self.model = decoder
1220
+
1221
+ def get_decoder(self):
1222
+ return self.model
1223
+
1224
+ @add_start_docstrings_to_model_forward(Deepseek_INPUTS_DOCSTRING)
1225
+ @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
1226
+ def forward(
1227
+ self,
1228
+ input_ids: torch.LongTensor = None,
1229
+ attention_mask: Optional[torch.Tensor] = None,
1230
+ position_ids: Optional[torch.LongTensor] = None,
1231
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
1232
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1233
+ labels: Optional[torch.LongTensor] = None,
1234
+ use_cache: Optional[bool] = None,
1235
+ output_attentions: Optional[bool] = None,
1236
+ output_hidden_states: Optional[bool] = None,
1237
+ return_dict: Optional[bool] = None,
1238
+ cache_position: Optional[torch.LongTensor] = None,
1239
+ num_logits_to_keep: int = 0,
1240
+ **kwargs: Unpack[KwargsForCausalLM],
1241
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
1242
+ r"""
1243
+ Args:
1244
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1245
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
1246
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
1247
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
1248
+
1249
+ num_logits_to_keep (`int`, *optional*):
1250
+ Calculate logits for the last `num_logits_to_keep` tokens. If `0`, calculate logits for all
1251
+ `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
1252
+ token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
1253
+
1254
+ Returns:
1255
+
1256
+ Example:
1257
+
1258
+ ```python
1259
+ >>> from transformers import AutoTokenizer
1260
+
1261
+ >>> model = DeepseekForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
1262
+ >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
1263
+
1264
+ >>> prompt = "Hey, are you conscious? Can you talk to me?"
1265
+ >>> inputs = tokenizer(prompt, return_tensors="pt")
1266
+
1267
+ >>> # Generate
1268
+ >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
1269
+ >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
1270
+ "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
1271
+ ```"""
1272
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
1273
+ output_hidden_states = (
1274
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1275
+ )
1276
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1277
+
1278
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
1279
+ outputs = self.model(
1280
+ input_ids=input_ids,
1281
+ attention_mask=attention_mask,
1282
+ position_ids=position_ids,
1283
+ past_key_values=past_key_values,
1284
+ inputs_embeds=inputs_embeds,
1285
+ use_cache=use_cache,
1286
+ output_attentions=output_attentions,
1287
+ output_hidden_states=output_hidden_states,
1288
+ return_dict=return_dict,
1289
+ )
1290
+
1291
+ hidden_states = outputs[0]
1292
+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
1293
+ logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :])
1294
+
1295
+ loss = None
1296
+ if labels is not None:
1297
+ loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
1298
+
1299
+ if not return_dict:
1300
+ output = (logits,) + outputs[1:]
1301
+ return (loss,) + output if loss is not None else output
1302
+
1303
+ return CausalLMOutputWithPast(
1304
+ loss=loss,
1305
+ logits=logits,
1306
+ past_key_values=outputs.past_key_values,
1307
+ hidden_states=outputs.hidden_states,
1308
+ attentions=outputs.attentions,
1309
+ )
1310
+
1311
+
1312
+ @add_start_docstrings(
1313
+ """
1314
+ The Deepseek Model transformer with a sequence classification head on top (linear layer).
1315
+
1316
+ [`DeepseekForSequenceClassification`] uses the last token in order to do the classification, as other causal models
1317
+ (e.g. GPT-2) do.
1318
+
1319
+ Since it does classification on the last token, it requires to know the position of the last token. If a
1320
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
1321
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
1322
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
1323
+ each row of the batch).
1324
+ """,
1325
+ Deepseek_START_DOCSTRING,
1326
+ )
1327
+ class DeepseekForSequenceClassification(DeepseekPreTrainedModel):
1328
+ def __init__(self, config):
1329
+ super().__init__(config)
1330
+ self.num_labels = config.num_labels
1331
+ self.model = DeepseekModel(config)
1332
+ self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
1333
+
1334
+ # Initialize weights and apply final processing
1335
+ self.post_init()
1336
+
1337
+ def get_input_embeddings(self):
1338
+ return self.model.embed_tokens
1339
+
1340
+ def set_input_embeddings(self, value):
1341
+ self.model.embed_tokens = value
1342
+
1343
+ @add_start_docstrings_to_model_forward(Deepseek_INPUTS_DOCSTRING)
1344
+ def forward(
1345
+ self,
1346
+ input_ids: Optional[torch.LongTensor] = None,
1347
+ attention_mask: Optional[torch.Tensor] = None,
1348
+ position_ids: Optional[torch.LongTensor] = None,
1349
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
1350
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1351
+ labels: Optional[torch.LongTensor] = None,
1352
+ use_cache: Optional[bool] = None,
1353
+ output_attentions: Optional[bool] = None,
1354
+ output_hidden_states: Optional[bool] = None,
1355
+ return_dict: Optional[bool] = None,
1356
+ ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
1357
+ r"""
1358
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1359
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1360
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1361
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1362
+ """
1363
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1364
+
1365
+ transformer_outputs = self.model(
1366
+ input_ids,
1367
+ attention_mask=attention_mask,
1368
+ position_ids=position_ids,
1369
+ past_key_values=past_key_values,
1370
+ inputs_embeds=inputs_embeds,
1371
+ use_cache=use_cache,
1372
+ output_attentions=output_attentions,
1373
+ output_hidden_states=output_hidden_states,
1374
+ return_dict=return_dict,
1375
+ )
1376
+ hidden_states = transformer_outputs[0]
1377
+ logits = self.score(hidden_states)
1378
+
1379
+ if input_ids is not None:
1380
+ batch_size = input_ids.shape[0]
1381
+ else:
1382
+ batch_size = inputs_embeds.shape[0]
1383
+
1384
+ if self.config.pad_token_id is None and batch_size != 1:
1385
+ raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
1386
+ if self.config.pad_token_id is None:
1387
+ sequence_lengths = -1
1388
+ else:
1389
+ if input_ids is not None:
1390
+ # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
1391
+ sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
1392
+ sequence_lengths = sequence_lengths % input_ids.shape[-1]
1393
+ sequence_lengths = sequence_lengths.to(logits.device)
1394
+ else:
1395
+ sequence_lengths = -1
1396
+
1397
+ pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
1398
+
1399
+ loss = None
1400
+ if labels is not None:
1401
+ loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config)
1402
+
1403
+ if not return_dict:
1404
+ output = (pooled_logits,) + transformer_outputs[1:]
1405
+ return ((loss,) + output) if loss is not None else output
1406
+
1407
+ return SequenceClassifierOutputWithPast(
1408
+ loss=loss,
1409
+ logits=pooled_logits,
1410
+ past_key_values=transformer_outputs.past_key_values,
1411
+ hidden_states=transformer_outputs.hidden_states,
1412
+ attentions=transformer_outputs.attentions,
1413
+ )
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "unk_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e405bd38ee048cfaadc76e093228e71de6af149bae73ff3547e6a578445bea6
3
+ size 10728072
tokenizer_config.json ADDED
@@ -0,0 +1,2084 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "128000": {
28
+ "content": "<|gigatoken_1|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "128001": {
36
+ "content": "<|gigatoken_2|>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "128002": {
44
+ "content": "<|gigatoken_3|>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "128003": {
52
+ "content": "<|gigatoken_4|>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "128004": {
60
+ "content": "<|gigatoken_5|>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "128005": {
68
+ "content": "<|gigatoken_6|>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "128006": {
76
+ "content": "<|gigatoken_7|>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "128007": {
84
+ "content": "<|gigatoken_8|>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "128008": {
92
+ "content": "<|gigatoken_9|>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "128009": {
100
+ "content": "<|gigatoken_10|>",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "128010": {
108
+ "content": "<|gigatoken_11|>",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "128011": {
116
+ "content": "<|gigatoken_12|>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "128012": {
124
+ "content": "<|gigatoken_13|>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "128013": {
132
+ "content": "<|gigatoken_14|>",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": true
138
+ },
139
+ "128014": {
140
+ "content": "<|gigatoken_15|>",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": true
146
+ },
147
+ "128015": {
148
+ "content": "<|gigatoken_16|>",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": true
154
+ },
155
+ "128016": {
156
+ "content": "<|gigatoken_17|>",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": true
162
+ },
163
+ "128017": {
164
+ "content": "<|gigatoken_18|>",
165
+ "lstrip": false,
166
+ "normalized": false,
167
+ "rstrip": false,
168
+ "single_word": false,
169
+ "special": true
170
+ },
171
+ "128018": {
172
+ "content": "<|gigatoken_19|>",
173
+ "lstrip": false,
174
+ "normalized": false,
175
+ "rstrip": false,
176
+ "single_word": false,
177
+ "special": true
178
+ },
179
+ "128019": {
180
+ "content": "<|gigatoken_20|>",
181
+ "lstrip": false,
182
+ "normalized": false,
183
+ "rstrip": false,
184
+ "single_word": false,
185
+ "special": true
186
+ },
187
+ "128020": {
188
+ "content": "<|gigatoken_21|>",
189
+ "lstrip": false,
190
+ "normalized": false,
191
+ "rstrip": false,
192
+ "single_word": false,
193
+ "special": true
194
+ },
195
+ "128021": {
196
+ "content": "<|gigatoken_22|>",
197
+ "lstrip": false,
198
+ "normalized": false,
199
+ "rstrip": false,
200
+ "single_word": false,
201
+ "special": true
202
+ },
203
+ "128022": {
204
+ "content": "<|gigatoken_23|>",
205
+ "lstrip": false,
206
+ "normalized": false,
207
+ "rstrip": false,
208
+ "single_word": false,
209
+ "special": true
210
+ },
211
+ "128023": {
212
+ "content": "<|gigatoken_24|>",
213
+ "lstrip": false,
214
+ "normalized": false,
215
+ "rstrip": false,
216
+ "single_word": false,
217
+ "special": true
218
+ },
219
+ "128024": {
220
+ "content": "<|gigatoken_25|>",
221
+ "lstrip": false,
222
+ "normalized": false,
223
+ "rstrip": false,
224
+ "single_word": false,
225
+ "special": true
226
+ },
227
+ "128025": {
228
+ "content": "<|gigatoken_26|>",
229
+ "lstrip": false,
230
+ "normalized": false,
231
+ "rstrip": false,
232
+ "single_word": false,
233
+ "special": true
234
+ },
235
+ "128026": {
236
+ "content": "<|gigatoken_27|>",
237
+ "lstrip": false,
238
+ "normalized": false,
239
+ "rstrip": false,
240
+ "single_word": false,
241
+ "special": true
242
+ },
243
+ "128027": {
244
+ "content": "<|gigatoken_28|>",
245
+ "lstrip": false,
246
+ "normalized": false,
247
+ "rstrip": false,
248
+ "single_word": false,
249
+ "special": true
250
+ },
251
+ "128028": {
252
+ "content": "<|gigatoken_29|>",
253
+ "lstrip": false,
254
+ "normalized": false,
255
+ "rstrip": false,
256
+ "single_word": false,
257
+ "special": true
258
+ },
259
+ "128029": {
260
+ "content": "<|gigatoken_30|>",
261
+ "lstrip": false,
262
+ "normalized": false,
263
+ "rstrip": false,
264
+ "single_word": false,
265
+ "special": true
266
+ },
267
+ "128030": {
268
+ "content": "<|gigatoken_31|>",
269
+ "lstrip": false,
270
+ "normalized": false,
271
+ "rstrip": false,
272
+ "single_word": false,
273
+ "special": true
274
+ },
275
+ "128031": {
276
+ "content": "<|gigatoken_32|>",
277
+ "lstrip": false,
278
+ "normalized": false,
279
+ "rstrip": false,
280
+ "single_word": false,
281
+ "special": true
282
+ },
283
+ "128032": {
284
+ "content": "<|gigatoken_33|>",
285
+ "lstrip": false,
286
+ "normalized": false,
287
+ "rstrip": false,
288
+ "single_word": false,
289
+ "special": true
290
+ },
291
+ "128033": {
292
+ "content": "<|gigatoken_34|>",
293
+ "lstrip": false,
294
+ "normalized": false,
295
+ "rstrip": false,
296
+ "single_word": false,
297
+ "special": true
298
+ },
299
+ "128034": {
300
+ "content": "<|gigatoken_35|>",
301
+ "lstrip": false,
302
+ "normalized": false,
303
+ "rstrip": false,
304
+ "single_word": false,
305
+ "special": true
306
+ },
307
+ "128035": {
308
+ "content": "<|gigatoken_36|>",
309
+ "lstrip": false,
310
+ "normalized": false,
311
+ "rstrip": false,
312
+ "single_word": false,
313
+ "special": true
314
+ },
315
+ "128036": {
316
+ "content": "<|gigatoken_37|>",
317
+ "lstrip": false,
318
+ "normalized": false,
319
+ "rstrip": false,
320
+ "single_word": false,
321
+ "special": true
322
+ },
323
+ "128037": {
324
+ "content": "<|gigatoken_38|>",
325
+ "lstrip": false,
326
+ "normalized": false,
327
+ "rstrip": false,
328
+ "single_word": false,
329
+ "special": true
330
+ },
331
+ "128038": {
332
+ "content": "<|gigatoken_39|>",
333
+ "lstrip": false,
334
+ "normalized": false,
335
+ "rstrip": false,
336
+ "single_word": false,
337
+ "special": true
338
+ },
339
+ "128039": {
340
+ "content": "<|gigatoken_40|>",
341
+ "lstrip": false,
342
+ "normalized": false,
343
+ "rstrip": false,
344
+ "single_word": false,
345
+ "special": true
346
+ },
347
+ "128040": {
348
+ "content": "<|gigatoken_41|>",
349
+ "lstrip": false,
350
+ "normalized": false,
351
+ "rstrip": false,
352
+ "single_word": false,
353
+ "special": true
354
+ },
355
+ "128041": {
356
+ "content": "<|gigatoken_42|>",
357
+ "lstrip": false,
358
+ "normalized": false,
359
+ "rstrip": false,
360
+ "single_word": false,
361
+ "special": true
362
+ },
363
+ "128042": {
364
+ "content": "<|gigatoken_43|>",
365
+ "lstrip": false,
366
+ "normalized": false,
367
+ "rstrip": false,
368
+ "single_word": false,
369
+ "special": true
370
+ },
371
+ "128043": {
372
+ "content": "<|gigatoken_44|>",
373
+ "lstrip": false,
374
+ "normalized": false,
375
+ "rstrip": false,
376
+ "single_word": false,
377
+ "special": true
378
+ },
379
+ "128044": {
380
+ "content": "<|gigatoken_45|>",
381
+ "lstrip": false,
382
+ "normalized": false,
383
+ "rstrip": false,
384
+ "single_word": false,
385
+ "special": true
386
+ },
387
+ "128045": {
388
+ "content": "<|gigatoken_46|>",
389
+ "lstrip": false,
390
+ "normalized": false,
391
+ "rstrip": false,
392
+ "single_word": false,
393
+ "special": true
394
+ },
395
+ "128046": {
396
+ "content": "<|gigatoken_47|>",
397
+ "lstrip": false,
398
+ "normalized": false,
399
+ "rstrip": false,
400
+ "single_word": false,
401
+ "special": true
402
+ },
403
+ "128047": {
404
+ "content": "<|gigatoken_48|>",
405
+ "lstrip": false,
406
+ "normalized": false,
407
+ "rstrip": false,
408
+ "single_word": false,
409
+ "special": true
410
+ },
411
+ "128048": {
412
+ "content": "<|gigatoken_49|>",
413
+ "lstrip": false,
414
+ "normalized": false,
415
+ "rstrip": false,
416
+ "single_word": false,
417
+ "special": true
418
+ },
419
+ "128049": {
420
+ "content": "<|gigatoken_50|>",
421
+ "lstrip": false,
422
+ "normalized": false,
423
+ "rstrip": false,
424
+ "single_word": false,
425
+ "special": true
426
+ },
427
+ "128050": {
428
+ "content": "<|gigatoken_51|>",
429
+ "lstrip": false,
430
+ "normalized": false,
431
+ "rstrip": false,
432
+ "single_word": false,
433
+ "special": true
434
+ },
435
+ "128051": {
436
+ "content": "<|gigatoken_52|>",
437
+ "lstrip": false,
438
+ "normalized": false,
439
+ "rstrip": false,
440
+ "single_word": false,
441
+ "special": true
442
+ },
443
+ "128052": {
444
+ "content": "<|gigatoken_53|>",
445
+ "lstrip": false,
446
+ "normalized": false,
447
+ "rstrip": false,
448
+ "single_word": false,
449
+ "special": true
450
+ },
451
+ "128053": {
452
+ "content": "<|gigatoken_54|>",
453
+ "lstrip": false,
454
+ "normalized": false,
455
+ "rstrip": false,
456
+ "single_word": false,
457
+ "special": true
458
+ },
459
+ "128054": {
460
+ "content": "<|gigatoken_55|>",
461
+ "lstrip": false,
462
+ "normalized": false,
463
+ "rstrip": false,
464
+ "single_word": false,
465
+ "special": true
466
+ },
467
+ "128055": {
468
+ "content": "<|gigatoken_56|>",
469
+ "lstrip": false,
470
+ "normalized": false,
471
+ "rstrip": false,
472
+ "single_word": false,
473
+ "special": true
474
+ },
475
+ "128056": {
476
+ "content": "<|gigatoken_57|>",
477
+ "lstrip": false,
478
+ "normalized": false,
479
+ "rstrip": false,
480
+ "single_word": false,
481
+ "special": true
482
+ },
483
+ "128057": {
484
+ "content": "<|gigatoken_58|>",
485
+ "lstrip": false,
486
+ "normalized": false,
487
+ "rstrip": false,
488
+ "single_word": false,
489
+ "special": true
490
+ },
491
+ "128058": {
492
+ "content": "<|gigatoken_59|>",
493
+ "lstrip": false,
494
+ "normalized": false,
495
+ "rstrip": false,
496
+ "single_word": false,
497
+ "special": true
498
+ },
499
+ "128059": {
500
+ "content": "<|gigatoken_60|>",
501
+ "lstrip": false,
502
+ "normalized": false,
503
+ "rstrip": false,
504
+ "single_word": false,
505
+ "special": true
506
+ },
507
+ "128060": {
508
+ "content": "<|gigatoken_61|>",
509
+ "lstrip": false,
510
+ "normalized": false,
511
+ "rstrip": false,
512
+ "single_word": false,
513
+ "special": true
514
+ },
515
+ "128061": {
516
+ "content": "<|gigatoken_62|>",
517
+ "lstrip": false,
518
+ "normalized": false,
519
+ "rstrip": false,
520
+ "single_word": false,
521
+ "special": true
522
+ },
523
+ "128062": {
524
+ "content": "<|gigatoken_63|>",
525
+ "lstrip": false,
526
+ "normalized": false,
527
+ "rstrip": false,
528
+ "single_word": false,
529
+ "special": true
530
+ },
531
+ "128063": {
532
+ "content": "<|gigatoken_64|>",
533
+ "lstrip": false,
534
+ "normalized": false,
535
+ "rstrip": false,
536
+ "single_word": false,
537
+ "special": true
538
+ },
539
+ "128064": {
540
+ "content": "<|gigatoken_65|>",
541
+ "lstrip": false,
542
+ "normalized": false,
543
+ "rstrip": false,
544
+ "single_word": false,
545
+ "special": true
546
+ },
547
+ "128065": {
548
+ "content": "<|gigatoken_66|>",
549
+ "lstrip": false,
550
+ "normalized": false,
551
+ "rstrip": false,
552
+ "single_word": false,
553
+ "special": true
554
+ },
555
+ "128066": {
556
+ "content": "<|gigatoken_67|>",
557
+ "lstrip": false,
558
+ "normalized": false,
559
+ "rstrip": false,
560
+ "single_word": false,
561
+ "special": true
562
+ },
563
+ "128067": {
564
+ "content": "<|gigatoken_68|>",
565
+ "lstrip": false,
566
+ "normalized": false,
567
+ "rstrip": false,
568
+ "single_word": false,
569
+ "special": true
570
+ },
571
+ "128068": {
572
+ "content": "<|gigatoken_69|>",
573
+ "lstrip": false,
574
+ "normalized": false,
575
+ "rstrip": false,
576
+ "single_word": false,
577
+ "special": true
578
+ },
579
+ "128069": {
580
+ "content": "<|gigatoken_70|>",
581
+ "lstrip": false,
582
+ "normalized": false,
583
+ "rstrip": false,
584
+ "single_word": false,
585
+ "special": true
586
+ },
587
+ "128070": {
588
+ "content": "<|gigatoken_71|>",
589
+ "lstrip": false,
590
+ "normalized": false,
591
+ "rstrip": false,
592
+ "single_word": false,
593
+ "special": true
594
+ },
595
+ "128071": {
596
+ "content": "<|gigatoken_72|>",
597
+ "lstrip": false,
598
+ "normalized": false,
599
+ "rstrip": false,
600
+ "single_word": false,
601
+ "special": true
602
+ },
603
+ "128072": {
604
+ "content": "<|gigatoken_73|>",
605
+ "lstrip": false,
606
+ "normalized": false,
607
+ "rstrip": false,
608
+ "single_word": false,
609
+ "special": true
610
+ },
611
+ "128073": {
612
+ "content": "<|gigatoken_74|>",
613
+ "lstrip": false,
614
+ "normalized": false,
615
+ "rstrip": false,
616
+ "single_word": false,
617
+ "special": true
618
+ },
619
+ "128074": {
620
+ "content": "<|gigatoken_75|>",
621
+ "lstrip": false,
622
+ "normalized": false,
623
+ "rstrip": false,
624
+ "single_word": false,
625
+ "special": true
626
+ },
627
+ "128075": {
628
+ "content": "<|gigatoken_76|>",
629
+ "lstrip": false,
630
+ "normalized": false,
631
+ "rstrip": false,
632
+ "single_word": false,
633
+ "special": true
634
+ },
635
+ "128076": {
636
+ "content": "<|gigatoken_77|>",
637
+ "lstrip": false,
638
+ "normalized": false,
639
+ "rstrip": false,
640
+ "single_word": false,
641
+ "special": true
642
+ },
643
+ "128077": {
644
+ "content": "<|gigatoken_78|>",
645
+ "lstrip": false,
646
+ "normalized": false,
647
+ "rstrip": false,
648
+ "single_word": false,
649
+ "special": true
650
+ },
651
+ "128078": {
652
+ "content": "<|gigatoken_79|>",
653
+ "lstrip": false,
654
+ "normalized": false,
655
+ "rstrip": false,
656
+ "single_word": false,
657
+ "special": true
658
+ },
659
+ "128079": {
660
+ "content": "<|gigatoken_80|>",
661
+ "lstrip": false,
662
+ "normalized": false,
663
+ "rstrip": false,
664
+ "single_word": false,
665
+ "special": true
666
+ },
667
+ "128080": {
668
+ "content": "<|gigatoken_81|>",
669
+ "lstrip": false,
670
+ "normalized": false,
671
+ "rstrip": false,
672
+ "single_word": false,
673
+ "special": true
674
+ },
675
+ "128081": {
676
+ "content": "<|gigatoken_82|>",
677
+ "lstrip": false,
678
+ "normalized": false,
679
+ "rstrip": false,
680
+ "single_word": false,
681
+ "special": true
682
+ },
683
+ "128082": {
684
+ "content": "<|gigatoken_83|>",
685
+ "lstrip": false,
686
+ "normalized": false,
687
+ "rstrip": false,
688
+ "single_word": false,
689
+ "special": true
690
+ },
691
+ "128083": {
692
+ "content": "<|gigatoken_84|>",
693
+ "lstrip": false,
694
+ "normalized": false,
695
+ "rstrip": false,
696
+ "single_word": false,
697
+ "special": true
698
+ },
699
+ "128084": {
700
+ "content": "<|gigatoken_85|>",
701
+ "lstrip": false,
702
+ "normalized": false,
703
+ "rstrip": false,
704
+ "single_word": false,
705
+ "special": true
706
+ },
707
+ "128085": {
708
+ "content": "<|gigatoken_86|>",
709
+ "lstrip": false,
710
+ "normalized": false,
711
+ "rstrip": false,
712
+ "single_word": false,
713
+ "special": true
714
+ },
715
+ "128086": {
716
+ "content": "<|gigatoken_87|>",
717
+ "lstrip": false,
718
+ "normalized": false,
719
+ "rstrip": false,
720
+ "single_word": false,
721
+ "special": true
722
+ },
723
+ "128087": {
724
+ "content": "<|gigatoken_88|>",
725
+ "lstrip": false,
726
+ "normalized": false,
727
+ "rstrip": false,
728
+ "single_word": false,
729
+ "special": true
730
+ },
731
+ "128088": {
732
+ "content": "<|gigatoken_89|>",
733
+ "lstrip": false,
734
+ "normalized": false,
735
+ "rstrip": false,
736
+ "single_word": false,
737
+ "special": true
738
+ },
739
+ "128089": {
740
+ "content": "<|gigatoken_90|>",
741
+ "lstrip": false,
742
+ "normalized": false,
743
+ "rstrip": false,
744
+ "single_word": false,
745
+ "special": true
746
+ },
747
+ "128090": {
748
+ "content": "<|gigatoken_91|>",
749
+ "lstrip": false,
750
+ "normalized": false,
751
+ "rstrip": false,
752
+ "single_word": false,
753
+ "special": true
754
+ },
755
+ "128091": {
756
+ "content": "<|gigatoken_92|>",
757
+ "lstrip": false,
758
+ "normalized": false,
759
+ "rstrip": false,
760
+ "single_word": false,
761
+ "special": true
762
+ },
763
+ "128092": {
764
+ "content": "<|gigatoken_93|>",
765
+ "lstrip": false,
766
+ "normalized": false,
767
+ "rstrip": false,
768
+ "single_word": false,
769
+ "special": true
770
+ },
771
+ "128093": {
772
+ "content": "<|gigatoken_94|>",
773
+ "lstrip": false,
774
+ "normalized": false,
775
+ "rstrip": false,
776
+ "single_word": false,
777
+ "special": true
778
+ },
779
+ "128094": {
780
+ "content": "<|gigatoken_95|>",
781
+ "lstrip": false,
782
+ "normalized": false,
783
+ "rstrip": false,
784
+ "single_word": false,
785
+ "special": true
786
+ },
787
+ "128095": {
788
+ "content": "<|gigatoken_96|>",
789
+ "lstrip": false,
790
+ "normalized": false,
791
+ "rstrip": false,
792
+ "single_word": false,
793
+ "special": true
794
+ },
795
+ "128096": {
796
+ "content": "<|gigatoken_97|>",
797
+ "lstrip": false,
798
+ "normalized": false,
799
+ "rstrip": false,
800
+ "single_word": false,
801
+ "special": true
802
+ },
803
+ "128097": {
804
+ "content": "<|gigatoken_98|>",
805
+ "lstrip": false,
806
+ "normalized": false,
807
+ "rstrip": false,
808
+ "single_word": false,
809
+ "special": true
810
+ },
811
+ "128098": {
812
+ "content": "<|gigatoken_99|>",
813
+ "lstrip": false,
814
+ "normalized": false,
815
+ "rstrip": false,
816
+ "single_word": false,
817
+ "special": true
818
+ },
819
+ "128099": {
820
+ "content": "<|gigatoken_100|>",
821
+ "lstrip": false,
822
+ "normalized": false,
823
+ "rstrip": false,
824
+ "single_word": false,
825
+ "special": true
826
+ },
827
+ "128100": {
828
+ "content": "<|gigatoken_101|>",
829
+ "lstrip": false,
830
+ "normalized": false,
831
+ "rstrip": false,
832
+ "single_word": false,
833
+ "special": true
834
+ },
835
+ "128101": {
836
+ "content": "<|gigatoken_102|>",
837
+ "lstrip": false,
838
+ "normalized": false,
839
+ "rstrip": false,
840
+ "single_word": false,
841
+ "special": true
842
+ },
843
+ "128102": {
844
+ "content": "<|gigatoken_103|>",
845
+ "lstrip": false,
846
+ "normalized": false,
847
+ "rstrip": false,
848
+ "single_word": false,
849
+ "special": true
850
+ },
851
+ "128103": {
852
+ "content": "<|gigatoken_104|>",
853
+ "lstrip": false,
854
+ "normalized": false,
855
+ "rstrip": false,
856
+ "single_word": false,
857
+ "special": true
858
+ },
859
+ "128104": {
860
+ "content": "<|gigatoken_105|>",
861
+ "lstrip": false,
862
+ "normalized": false,
863
+ "rstrip": false,
864
+ "single_word": false,
865
+ "special": true
866
+ },
867
+ "128105": {
868
+ "content": "<|gigatoken_106|>",
869
+ "lstrip": false,
870
+ "normalized": false,
871
+ "rstrip": false,
872
+ "single_word": false,
873
+ "special": true
874
+ },
875
+ "128106": {
876
+ "content": "<|gigatoken_107|>",
877
+ "lstrip": false,
878
+ "normalized": false,
879
+ "rstrip": false,
880
+ "single_word": false,
881
+ "special": true
882
+ },
883
+ "128107": {
884
+ "content": "<|gigatoken_108|>",
885
+ "lstrip": false,
886
+ "normalized": false,
887
+ "rstrip": false,
888
+ "single_word": false,
889
+ "special": true
890
+ },
891
+ "128108": {
892
+ "content": "<|gigatoken_109|>",
893
+ "lstrip": false,
894
+ "normalized": false,
895
+ "rstrip": false,
896
+ "single_word": false,
897
+ "special": true
898
+ },
899
+ "128109": {
900
+ "content": "<|gigatoken_110|>",
901
+ "lstrip": false,
902
+ "normalized": false,
903
+ "rstrip": false,
904
+ "single_word": false,
905
+ "special": true
906
+ },
907
+ "128110": {
908
+ "content": "<|gigatoken_111|>",
909
+ "lstrip": false,
910
+ "normalized": false,
911
+ "rstrip": false,
912
+ "single_word": false,
913
+ "special": true
914
+ },
915
+ "128111": {
916
+ "content": "<|gigatoken_112|>",
917
+ "lstrip": false,
918
+ "normalized": false,
919
+ "rstrip": false,
920
+ "single_word": false,
921
+ "special": true
922
+ },
923
+ "128112": {
924
+ "content": "<|gigatoken_113|>",
925
+ "lstrip": false,
926
+ "normalized": false,
927
+ "rstrip": false,
928
+ "single_word": false,
929
+ "special": true
930
+ },
931
+ "128113": {
932
+ "content": "<|gigatoken_114|>",
933
+ "lstrip": false,
934
+ "normalized": false,
935
+ "rstrip": false,
936
+ "single_word": false,
937
+ "special": true
938
+ },
939
+ "128114": {
940
+ "content": "<|gigatoken_115|>",
941
+ "lstrip": false,
942
+ "normalized": false,
943
+ "rstrip": false,
944
+ "single_word": false,
945
+ "special": true
946
+ },
947
+ "128115": {
948
+ "content": "<|gigatoken_116|>",
949
+ "lstrip": false,
950
+ "normalized": false,
951
+ "rstrip": false,
952
+ "single_word": false,
953
+ "special": true
954
+ },
955
+ "128116": {
956
+ "content": "<|gigatoken_117|>",
957
+ "lstrip": false,
958
+ "normalized": false,
959
+ "rstrip": false,
960
+ "single_word": false,
961
+ "special": true
962
+ },
963
+ "128117": {
964
+ "content": "<|gigatoken_118|>",
965
+ "lstrip": false,
966
+ "normalized": false,
967
+ "rstrip": false,
968
+ "single_word": false,
969
+ "special": true
970
+ },
971
+ "128118": {
972
+ "content": "<|gigatoken_119|>",
973
+ "lstrip": false,
974
+ "normalized": false,
975
+ "rstrip": false,
976
+ "single_word": false,
977
+ "special": true
978
+ },
979
+ "128119": {
980
+ "content": "<|gigatoken_120|>",
981
+ "lstrip": false,
982
+ "normalized": false,
983
+ "rstrip": false,
984
+ "single_word": false,
985
+ "special": true
986
+ },
987
+ "128120": {
988
+ "content": "<|gigatoken_121|>",
989
+ "lstrip": false,
990
+ "normalized": false,
991
+ "rstrip": false,
992
+ "single_word": false,
993
+ "special": true
994
+ },
995
+ "128121": {
996
+ "content": "<|gigatoken_122|>",
997
+ "lstrip": false,
998
+ "normalized": false,
999
+ "rstrip": false,
1000
+ "single_word": false,
1001
+ "special": true
1002
+ },
1003
+ "128122": {
1004
+ "content": "<|gigatoken_123|>",
1005
+ "lstrip": false,
1006
+ "normalized": false,
1007
+ "rstrip": false,
1008
+ "single_word": false,
1009
+ "special": true
1010
+ },
1011
+ "128123": {
1012
+ "content": "<|gigatoken_124|>",
1013
+ "lstrip": false,
1014
+ "normalized": false,
1015
+ "rstrip": false,
1016
+ "single_word": false,
1017
+ "special": true
1018
+ },
1019
+ "128124": {
1020
+ "content": "<|gigatoken_125|>",
1021
+ "lstrip": false,
1022
+ "normalized": false,
1023
+ "rstrip": false,
1024
+ "single_word": false,
1025
+ "special": true
1026
+ },
1027
+ "128125": {
1028
+ "content": "<|gigatoken_126|>",
1029
+ "lstrip": false,
1030
+ "normalized": false,
1031
+ "rstrip": false,
1032
+ "single_word": false,
1033
+ "special": true
1034
+ },
1035
+ "128126": {
1036
+ "content": "<|gigatoken_127|>",
1037
+ "lstrip": false,
1038
+ "normalized": false,
1039
+ "rstrip": false,
1040
+ "single_word": false,
1041
+ "special": true
1042
+ },
1043
+ "128127": {
1044
+ "content": "<|gigatoken_128|>",
1045
+ "lstrip": false,
1046
+ "normalized": false,
1047
+ "rstrip": false,
1048
+ "single_word": false,
1049
+ "special": true
1050
+ },
1051
+ "128128": {
1052
+ "content": "<|gigatoken_129|>",
1053
+ "lstrip": false,
1054
+ "normalized": false,
1055
+ "rstrip": false,
1056
+ "single_word": false,
1057
+ "special": true
1058
+ },
1059
+ "128129": {
1060
+ "content": "<|gigatoken_130|>",
1061
+ "lstrip": false,
1062
+ "normalized": false,
1063
+ "rstrip": false,
1064
+ "single_word": false,
1065
+ "special": true
1066
+ },
1067
+ "128130": {
1068
+ "content": "<|gigatoken_131|>",
1069
+ "lstrip": false,
1070
+ "normalized": false,
1071
+ "rstrip": false,
1072
+ "single_word": false,
1073
+ "special": true
1074
+ },
1075
+ "128131": {
1076
+ "content": "<|gigatoken_132|>",
1077
+ "lstrip": false,
1078
+ "normalized": false,
1079
+ "rstrip": false,
1080
+ "single_word": false,
1081
+ "special": true
1082
+ },
1083
+ "128132": {
1084
+ "content": "<|gigatoken_133|>",
1085
+ "lstrip": false,
1086
+ "normalized": false,
1087
+ "rstrip": false,
1088
+ "single_word": false,
1089
+ "special": true
1090
+ },
1091
+ "128133": {
1092
+ "content": "<|gigatoken_134|>",
1093
+ "lstrip": false,
1094
+ "normalized": false,
1095
+ "rstrip": false,
1096
+ "single_word": false,
1097
+ "special": true
1098
+ },
1099
+ "128134": {
1100
+ "content": "<|gigatoken_135|>",
1101
+ "lstrip": false,
1102
+ "normalized": false,
1103
+ "rstrip": false,
1104
+ "single_word": false,
1105
+ "special": true
1106
+ },
1107
+ "128135": {
1108
+ "content": "<|gigatoken_136|>",
1109
+ "lstrip": false,
1110
+ "normalized": false,
1111
+ "rstrip": false,
1112
+ "single_word": false,
1113
+ "special": true
1114
+ },
1115
+ "128136": {
1116
+ "content": "<|gigatoken_137|>",
1117
+ "lstrip": false,
1118
+ "normalized": false,
1119
+ "rstrip": false,
1120
+ "single_word": false,
1121
+ "special": true
1122
+ },
1123
+ "128137": {
1124
+ "content": "<|gigatoken_138|>",
1125
+ "lstrip": false,
1126
+ "normalized": false,
1127
+ "rstrip": false,
1128
+ "single_word": false,
1129
+ "special": true
1130
+ },
1131
+ "128138": {
1132
+ "content": "<|gigatoken_139|>",
1133
+ "lstrip": false,
1134
+ "normalized": false,
1135
+ "rstrip": false,
1136
+ "single_word": false,
1137
+ "special": true
1138
+ },
1139
+ "128139": {
1140
+ "content": "<|gigatoken_140|>",
1141
+ "lstrip": false,
1142
+ "normalized": false,
1143
+ "rstrip": false,
1144
+ "single_word": false,
1145
+ "special": true
1146
+ },
1147
+ "128140": {
1148
+ "content": "<|gigatoken_141|>",
1149
+ "lstrip": false,
1150
+ "normalized": false,
1151
+ "rstrip": false,
1152
+ "single_word": false,
1153
+ "special": true
1154
+ },
1155
+ "128141": {
1156
+ "content": "<|gigatoken_142|>",
1157
+ "lstrip": false,
1158
+ "normalized": false,
1159
+ "rstrip": false,
1160
+ "single_word": false,
1161
+ "special": true
1162
+ },
1163
+ "128142": {
1164
+ "content": "<|gigatoken_143|>",
1165
+ "lstrip": false,
1166
+ "normalized": false,
1167
+ "rstrip": false,
1168
+ "single_word": false,
1169
+ "special": true
1170
+ },
1171
+ "128143": {
1172
+ "content": "<|gigatoken_144|>",
1173
+ "lstrip": false,
1174
+ "normalized": false,
1175
+ "rstrip": false,
1176
+ "single_word": false,
1177
+ "special": true
1178
+ },
1179
+ "128144": {
1180
+ "content": "<|gigatoken_145|>",
1181
+ "lstrip": false,
1182
+ "normalized": false,
1183
+ "rstrip": false,
1184
+ "single_word": false,
1185
+ "special": true
1186
+ },
1187
+ "128145": {
1188
+ "content": "<|gigatoken_146|>",
1189
+ "lstrip": false,
1190
+ "normalized": false,
1191
+ "rstrip": false,
1192
+ "single_word": false,
1193
+ "special": true
1194
+ },
1195
+ "128146": {
1196
+ "content": "<|gigatoken_147|>",
1197
+ "lstrip": false,
1198
+ "normalized": false,
1199
+ "rstrip": false,
1200
+ "single_word": false,
1201
+ "special": true
1202
+ },
1203
+ "128147": {
1204
+ "content": "<|gigatoken_148|>",
1205
+ "lstrip": false,
1206
+ "normalized": false,
1207
+ "rstrip": false,
1208
+ "single_word": false,
1209
+ "special": true
1210
+ },
1211
+ "128148": {
1212
+ "content": "<|gigatoken_149|>",
1213
+ "lstrip": false,
1214
+ "normalized": false,
1215
+ "rstrip": false,
1216
+ "single_word": false,
1217
+ "special": true
1218
+ },
1219
+ "128149": {
1220
+ "content": "<|gigatoken_150|>",
1221
+ "lstrip": false,
1222
+ "normalized": false,
1223
+ "rstrip": false,
1224
+ "single_word": false,
1225
+ "special": true
1226
+ },
1227
+ "128150": {
1228
+ "content": "<|gigatoken_151|>",
1229
+ "lstrip": false,
1230
+ "normalized": false,
1231
+ "rstrip": false,
1232
+ "single_word": false,
1233
+ "special": true
1234
+ },
1235
+ "128151": {
1236
+ "content": "<|gigatoken_152|>",
1237
+ "lstrip": false,
1238
+ "normalized": false,
1239
+ "rstrip": false,
1240
+ "single_word": false,
1241
+ "special": true
1242
+ },
1243
+ "128152": {
1244
+ "content": "<|gigatoken_153|>",
1245
+ "lstrip": false,
1246
+ "normalized": false,
1247
+ "rstrip": false,
1248
+ "single_word": false,
1249
+ "special": true
1250
+ },
1251
+ "128153": {
1252
+ "content": "<|gigatoken_154|>",
1253
+ "lstrip": false,
1254
+ "normalized": false,
1255
+ "rstrip": false,
1256
+ "single_word": false,
1257
+ "special": true
1258
+ },
1259
+ "128154": {
1260
+ "content": "<|gigatoken_155|>",
1261
+ "lstrip": false,
1262
+ "normalized": false,
1263
+ "rstrip": false,
1264
+ "single_word": false,
1265
+ "special": true
1266
+ },
1267
+ "128155": {
1268
+ "content": "<|gigatoken_156|>",
1269
+ "lstrip": false,
1270
+ "normalized": false,
1271
+ "rstrip": false,
1272
+ "single_word": false,
1273
+ "special": true
1274
+ },
1275
+ "128156": {
1276
+ "content": "<|gigatoken_157|>",
1277
+ "lstrip": false,
1278
+ "normalized": false,
1279
+ "rstrip": false,
1280
+ "single_word": false,
1281
+ "special": true
1282
+ },
1283
+ "128157": {
1284
+ "content": "<|gigatoken_158|>",
1285
+ "lstrip": false,
1286
+ "normalized": false,
1287
+ "rstrip": false,
1288
+ "single_word": false,
1289
+ "special": true
1290
+ },
1291
+ "128158": {
1292
+ "content": "<|gigatoken_159|>",
1293
+ "lstrip": false,
1294
+ "normalized": false,
1295
+ "rstrip": false,
1296
+ "single_word": false,
1297
+ "special": true
1298
+ },
1299
+ "128159": {
1300
+ "content": "<|gigatoken_160|>",
1301
+ "lstrip": false,
1302
+ "normalized": false,
1303
+ "rstrip": false,
1304
+ "single_word": false,
1305
+ "special": true
1306
+ },
1307
+ "128160": {
1308
+ "content": "<|gigatoken_161|>",
1309
+ "lstrip": false,
1310
+ "normalized": false,
1311
+ "rstrip": false,
1312
+ "single_word": false,
1313
+ "special": true
1314
+ },
1315
+ "128161": {
1316
+ "content": "<|gigatoken_162|>",
1317
+ "lstrip": false,
1318
+ "normalized": false,
1319
+ "rstrip": false,
1320
+ "single_word": false,
1321
+ "special": true
1322
+ },
1323
+ "128162": {
1324
+ "content": "<|gigatoken_163|>",
1325
+ "lstrip": false,
1326
+ "normalized": false,
1327
+ "rstrip": false,
1328
+ "single_word": false,
1329
+ "special": true
1330
+ },
1331
+ "128163": {
1332
+ "content": "<|gigatoken_164|>",
1333
+ "lstrip": false,
1334
+ "normalized": false,
1335
+ "rstrip": false,
1336
+ "single_word": false,
1337
+ "special": true
1338
+ },
1339
+ "128164": {
1340
+ "content": "<|gigatoken_165|>",
1341
+ "lstrip": false,
1342
+ "normalized": false,
1343
+ "rstrip": false,
1344
+ "single_word": false,
1345
+ "special": true
1346
+ },
1347
+ "128165": {
1348
+ "content": "<|gigatoken_166|>",
1349
+ "lstrip": false,
1350
+ "normalized": false,
1351
+ "rstrip": false,
1352
+ "single_word": false,
1353
+ "special": true
1354
+ },
1355
+ "128166": {
1356
+ "content": "<|gigatoken_167|>",
1357
+ "lstrip": false,
1358
+ "normalized": false,
1359
+ "rstrip": false,
1360
+ "single_word": false,
1361
+ "special": true
1362
+ },
1363
+ "128167": {
1364
+ "content": "<|gigatoken_168|>",
1365
+ "lstrip": false,
1366
+ "normalized": false,
1367
+ "rstrip": false,
1368
+ "single_word": false,
1369
+ "special": true
1370
+ },
1371
+ "128168": {
1372
+ "content": "<|gigatoken_169|>",
1373
+ "lstrip": false,
1374
+ "normalized": false,
1375
+ "rstrip": false,
1376
+ "single_word": false,
1377
+ "special": true
1378
+ },
1379
+ "128169": {
1380
+ "content": "<|gigatoken_170|>",
1381
+ "lstrip": false,
1382
+ "normalized": false,
1383
+ "rstrip": false,
1384
+ "single_word": false,
1385
+ "special": true
1386
+ },
1387
+ "128170": {
1388
+ "content": "<|gigatoken_171|>",
1389
+ "lstrip": false,
1390
+ "normalized": false,
1391
+ "rstrip": false,
1392
+ "single_word": false,
1393
+ "special": true
1394
+ },
1395
+ "128171": {
1396
+ "content": "<|gigatoken_172|>",
1397
+ "lstrip": false,
1398
+ "normalized": false,
1399
+ "rstrip": false,
1400
+ "single_word": false,
1401
+ "special": true
1402
+ },
1403
+ "128172": {
1404
+ "content": "<|gigatoken_173|>",
1405
+ "lstrip": false,
1406
+ "normalized": false,
1407
+ "rstrip": false,
1408
+ "single_word": false,
1409
+ "special": true
1410
+ },
1411
+ "128173": {
1412
+ "content": "<|gigatoken_174|>",
1413
+ "lstrip": false,
1414
+ "normalized": false,
1415
+ "rstrip": false,
1416
+ "single_word": false,
1417
+ "special": true
1418
+ },
1419
+ "128174": {
1420
+ "content": "<|gigatoken_175|>",
1421
+ "lstrip": false,
1422
+ "normalized": false,
1423
+ "rstrip": false,
1424
+ "single_word": false,
1425
+ "special": true
1426
+ },
1427
+ "128175": {
1428
+ "content": "<|gigatoken_176|>",
1429
+ "lstrip": false,
1430
+ "normalized": false,
1431
+ "rstrip": false,
1432
+ "single_word": false,
1433
+ "special": true
1434
+ },
1435
+ "128176": {
1436
+ "content": "<|gigatoken_177|>",
1437
+ "lstrip": false,
1438
+ "normalized": false,
1439
+ "rstrip": false,
1440
+ "single_word": false,
1441
+ "special": true
1442
+ },
1443
+ "128177": {
1444
+ "content": "<|gigatoken_178|>",
1445
+ "lstrip": false,
1446
+ "normalized": false,
1447
+ "rstrip": false,
1448
+ "single_word": false,
1449
+ "special": true
1450
+ },
1451
+ "128178": {
1452
+ "content": "<|gigatoken_179|>",
1453
+ "lstrip": false,
1454
+ "normalized": false,
1455
+ "rstrip": false,
1456
+ "single_word": false,
1457
+ "special": true
1458
+ },
1459
+ "128179": {
1460
+ "content": "<|gigatoken_180|>",
1461
+ "lstrip": false,
1462
+ "normalized": false,
1463
+ "rstrip": false,
1464
+ "single_word": false,
1465
+ "special": true
1466
+ },
1467
+ "128180": {
1468
+ "content": "<|gigatoken_181|>",
1469
+ "lstrip": false,
1470
+ "normalized": false,
1471
+ "rstrip": false,
1472
+ "single_word": false,
1473
+ "special": true
1474
+ },
1475
+ "128181": {
1476
+ "content": "<|gigatoken_182|>",
1477
+ "lstrip": false,
1478
+ "normalized": false,
1479
+ "rstrip": false,
1480
+ "single_word": false,
1481
+ "special": true
1482
+ },
1483
+ "128182": {
1484
+ "content": "<|gigatoken_183|>",
1485
+ "lstrip": false,
1486
+ "normalized": false,
1487
+ "rstrip": false,
1488
+ "single_word": false,
1489
+ "special": true
1490
+ },
1491
+ "128183": {
1492
+ "content": "<|gigatoken_184|>",
1493
+ "lstrip": false,
1494
+ "normalized": false,
1495
+ "rstrip": false,
1496
+ "single_word": false,
1497
+ "special": true
1498
+ },
1499
+ "128184": {
1500
+ "content": "<|gigatoken_185|>",
1501
+ "lstrip": false,
1502
+ "normalized": false,
1503
+ "rstrip": false,
1504
+ "single_word": false,
1505
+ "special": true
1506
+ },
1507
+ "128185": {
1508
+ "content": "<|gigatoken_186|>",
1509
+ "lstrip": false,
1510
+ "normalized": false,
1511
+ "rstrip": false,
1512
+ "single_word": false,
1513
+ "special": true
1514
+ },
1515
+ "128186": {
1516
+ "content": "<|gigatoken_187|>",
1517
+ "lstrip": false,
1518
+ "normalized": false,
1519
+ "rstrip": false,
1520
+ "single_word": false,
1521
+ "special": true
1522
+ },
1523
+ "128187": {
1524
+ "content": "<|gigatoken_188|>",
1525
+ "lstrip": false,
1526
+ "normalized": false,
1527
+ "rstrip": false,
1528
+ "single_word": false,
1529
+ "special": true
1530
+ },
1531
+ "128188": {
1532
+ "content": "<|gigatoken_189|>",
1533
+ "lstrip": false,
1534
+ "normalized": false,
1535
+ "rstrip": false,
1536
+ "single_word": false,
1537
+ "special": true
1538
+ },
1539
+ "128189": {
1540
+ "content": "<|gigatoken_190|>",
1541
+ "lstrip": false,
1542
+ "normalized": false,
1543
+ "rstrip": false,
1544
+ "single_word": false,
1545
+ "special": true
1546
+ },
1547
+ "128190": {
1548
+ "content": "<|gigatoken_191|>",
1549
+ "lstrip": false,
1550
+ "normalized": false,
1551
+ "rstrip": false,
1552
+ "single_word": false,
1553
+ "special": true
1554
+ },
1555
+ "128191": {
1556
+ "content": "<|gigatoken_192|>",
1557
+ "lstrip": false,
1558
+ "normalized": false,
1559
+ "rstrip": false,
1560
+ "single_word": false,
1561
+ "special": true
1562
+ },
1563
+ "128192": {
1564
+ "content": "<|gigatoken_193|>",
1565
+ "lstrip": false,
1566
+ "normalized": false,
1567
+ "rstrip": false,
1568
+ "single_word": false,
1569
+ "special": true
1570
+ },
1571
+ "128193": {
1572
+ "content": "<|gigatoken_194|>",
1573
+ "lstrip": false,
1574
+ "normalized": false,
1575
+ "rstrip": false,
1576
+ "single_word": false,
1577
+ "special": true
1578
+ },
1579
+ "128194": {
1580
+ "content": "<|gigatoken_195|>",
1581
+ "lstrip": false,
1582
+ "normalized": false,
1583
+ "rstrip": false,
1584
+ "single_word": false,
1585
+ "special": true
1586
+ },
1587
+ "128195": {
1588
+ "content": "<|gigatoken_196|>",
1589
+ "lstrip": false,
1590
+ "normalized": false,
1591
+ "rstrip": false,
1592
+ "single_word": false,
1593
+ "special": true
1594
+ },
1595
+ "128196": {
1596
+ "content": "<|gigatoken_197|>",
1597
+ "lstrip": false,
1598
+ "normalized": false,
1599
+ "rstrip": false,
1600
+ "single_word": false,
1601
+ "special": true
1602
+ },
1603
+ "128197": {
1604
+ "content": "<|gigatoken_198|>",
1605
+ "lstrip": false,
1606
+ "normalized": false,
1607
+ "rstrip": false,
1608
+ "single_word": false,
1609
+ "special": true
1610
+ },
1611
+ "128198": {
1612
+ "content": "<|gigatoken_199|>",
1613
+ "lstrip": false,
1614
+ "normalized": false,
1615
+ "rstrip": false,
1616
+ "single_word": false,
1617
+ "special": true
1618
+ },
1619
+ "128199": {
1620
+ "content": "<|gigatoken_200|>",
1621
+ "lstrip": false,
1622
+ "normalized": false,
1623
+ "rstrip": false,
1624
+ "single_word": false,
1625
+ "special": true
1626
+ },
1627
+ "128200": {
1628
+ "content": "<|gigatoken_201|>",
1629
+ "lstrip": false,
1630
+ "normalized": false,
1631
+ "rstrip": false,
1632
+ "single_word": false,
1633
+ "special": true
1634
+ },
1635
+ "128201": {
1636
+ "content": "<|gigatoken_202|>",
1637
+ "lstrip": false,
1638
+ "normalized": false,
1639
+ "rstrip": false,
1640
+ "single_word": false,
1641
+ "special": true
1642
+ },
1643
+ "128202": {
1644
+ "content": "<|gigatoken_203|>",
1645
+ "lstrip": false,
1646
+ "normalized": false,
1647
+ "rstrip": false,
1648
+ "single_word": false,
1649
+ "special": true
1650
+ },
1651
+ "128203": {
1652
+ "content": "<|gigatoken_204|>",
1653
+ "lstrip": false,
1654
+ "normalized": false,
1655
+ "rstrip": false,
1656
+ "single_word": false,
1657
+ "special": true
1658
+ },
1659
+ "128204": {
1660
+ "content": "<|gigatoken_205|>",
1661
+ "lstrip": false,
1662
+ "normalized": false,
1663
+ "rstrip": false,
1664
+ "single_word": false,
1665
+ "special": true
1666
+ },
1667
+ "128205": {
1668
+ "content": "<|gigatoken_206|>",
1669
+ "lstrip": false,
1670
+ "normalized": false,
1671
+ "rstrip": false,
1672
+ "single_word": false,
1673
+ "special": true
1674
+ },
1675
+ "128206": {
1676
+ "content": "<|gigatoken_207|>",
1677
+ "lstrip": false,
1678
+ "normalized": false,
1679
+ "rstrip": false,
1680
+ "single_word": false,
1681
+ "special": true
1682
+ },
1683
+ "128207": {
1684
+ "content": "<|gigatoken_208|>",
1685
+ "lstrip": false,
1686
+ "normalized": false,
1687
+ "rstrip": false,
1688
+ "single_word": false,
1689
+ "special": true
1690
+ },
1691
+ "128208": {
1692
+ "content": "<|gigatoken_209|>",
1693
+ "lstrip": false,
1694
+ "normalized": false,
1695
+ "rstrip": false,
1696
+ "single_word": false,
1697
+ "special": true
1698
+ },
1699
+ "128209": {
1700
+ "content": "<|gigatoken_210|>",
1701
+ "lstrip": false,
1702
+ "normalized": false,
1703
+ "rstrip": false,
1704
+ "single_word": false,
1705
+ "special": true
1706
+ },
1707
+ "128210": {
1708
+ "content": "<|gigatoken_211|>",
1709
+ "lstrip": false,
1710
+ "normalized": false,
1711
+ "rstrip": false,
1712
+ "single_word": false,
1713
+ "special": true
1714
+ },
1715
+ "128211": {
1716
+ "content": "<|gigatoken_212|>",
1717
+ "lstrip": false,
1718
+ "normalized": false,
1719
+ "rstrip": false,
1720
+ "single_word": false,
1721
+ "special": true
1722
+ },
1723
+ "128212": {
1724
+ "content": "<|gigatoken_213|>",
1725
+ "lstrip": false,
1726
+ "normalized": false,
1727
+ "rstrip": false,
1728
+ "single_word": false,
1729
+ "special": true
1730
+ },
1731
+ "128213": {
1732
+ "content": "<|gigatoken_214|>",
1733
+ "lstrip": false,
1734
+ "normalized": false,
1735
+ "rstrip": false,
1736
+ "single_word": false,
1737
+ "special": true
1738
+ },
1739
+ "128214": {
1740
+ "content": "<|gigatoken_215|>",
1741
+ "lstrip": false,
1742
+ "normalized": false,
1743
+ "rstrip": false,
1744
+ "single_word": false,
1745
+ "special": true
1746
+ },
1747
+ "128215": {
1748
+ "content": "<|gigatoken_216|>",
1749
+ "lstrip": false,
1750
+ "normalized": false,
1751
+ "rstrip": false,
1752
+ "single_word": false,
1753
+ "special": true
1754
+ },
1755
+ "128216": {
1756
+ "content": "<|gigatoken_217|>",
1757
+ "lstrip": false,
1758
+ "normalized": false,
1759
+ "rstrip": false,
1760
+ "single_word": false,
1761
+ "special": true
1762
+ },
1763
+ "128217": {
1764
+ "content": "<|gigatoken_218|>",
1765
+ "lstrip": false,
1766
+ "normalized": false,
1767
+ "rstrip": false,
1768
+ "single_word": false,
1769
+ "special": true
1770
+ },
1771
+ "128218": {
1772
+ "content": "<|gigatoken_219|>",
1773
+ "lstrip": false,
1774
+ "normalized": false,
1775
+ "rstrip": false,
1776
+ "single_word": false,
1777
+ "special": true
1778
+ },
1779
+ "128219": {
1780
+ "content": "<|gigatoken_220|>",
1781
+ "lstrip": false,
1782
+ "normalized": false,
1783
+ "rstrip": false,
1784
+ "single_word": false,
1785
+ "special": true
1786
+ },
1787
+ "128220": {
1788
+ "content": "<|gigatoken_221|>",
1789
+ "lstrip": false,
1790
+ "normalized": false,
1791
+ "rstrip": false,
1792
+ "single_word": false,
1793
+ "special": true
1794
+ },
1795
+ "128221": {
1796
+ "content": "<|gigatoken_222|>",
1797
+ "lstrip": false,
1798
+ "normalized": false,
1799
+ "rstrip": false,
1800
+ "single_word": false,
1801
+ "special": true
1802
+ },
1803
+ "128222": {
1804
+ "content": "<|gigatoken_223|>",
1805
+ "lstrip": false,
1806
+ "normalized": false,
1807
+ "rstrip": false,
1808
+ "single_word": false,
1809
+ "special": true
1810
+ },
1811
+ "128223": {
1812
+ "content": "<|gigatoken_224|>",
1813
+ "lstrip": false,
1814
+ "normalized": false,
1815
+ "rstrip": false,
1816
+ "single_word": false,
1817
+ "special": true
1818
+ },
1819
+ "128224": {
1820
+ "content": "<|gigatoken_225|>",
1821
+ "lstrip": false,
1822
+ "normalized": false,
1823
+ "rstrip": false,
1824
+ "single_word": false,
1825
+ "special": true
1826
+ },
1827
+ "128225": {
1828
+ "content": "<|gigatoken_226|>",
1829
+ "lstrip": false,
1830
+ "normalized": false,
1831
+ "rstrip": false,
1832
+ "single_word": false,
1833
+ "special": true
1834
+ },
1835
+ "128226": {
1836
+ "content": "<|gigatoken_227|>",
1837
+ "lstrip": false,
1838
+ "normalized": false,
1839
+ "rstrip": false,
1840
+ "single_word": false,
1841
+ "special": true
1842
+ },
1843
+ "128227": {
1844
+ "content": "<|gigatoken_228|>",
1845
+ "lstrip": false,
1846
+ "normalized": false,
1847
+ "rstrip": false,
1848
+ "single_word": false,
1849
+ "special": true
1850
+ },
1851
+ "128228": {
1852
+ "content": "<|gigatoken_229|>",
1853
+ "lstrip": false,
1854
+ "normalized": false,
1855
+ "rstrip": false,
1856
+ "single_word": false,
1857
+ "special": true
1858
+ },
1859
+ "128229": {
1860
+ "content": "<|gigatoken_230|>",
1861
+ "lstrip": false,
1862
+ "normalized": false,
1863
+ "rstrip": false,
1864
+ "single_word": false,
1865
+ "special": true
1866
+ },
1867
+ "128230": {
1868
+ "content": "<|gigatoken_231|>",
1869
+ "lstrip": false,
1870
+ "normalized": false,
1871
+ "rstrip": false,
1872
+ "single_word": false,
1873
+ "special": true
1874
+ },
1875
+ "128231": {
1876
+ "content": "<|gigatoken_232|>",
1877
+ "lstrip": false,
1878
+ "normalized": false,
1879
+ "rstrip": false,
1880
+ "single_word": false,
1881
+ "special": true
1882
+ },
1883
+ "128232": {
1884
+ "content": "<|gigatoken_233|>",
1885
+ "lstrip": false,
1886
+ "normalized": false,
1887
+ "rstrip": false,
1888
+ "single_word": false,
1889
+ "special": true
1890
+ },
1891
+ "128233": {
1892
+ "content": "<|gigatoken_234|>",
1893
+ "lstrip": false,
1894
+ "normalized": false,
1895
+ "rstrip": false,
1896
+ "single_word": false,
1897
+ "special": true
1898
+ },
1899
+ "128234": {
1900
+ "content": "<|gigatoken_235|>",
1901
+ "lstrip": false,
1902
+ "normalized": false,
1903
+ "rstrip": false,
1904
+ "single_word": false,
1905
+ "special": true
1906
+ },
1907
+ "128235": {
1908
+ "content": "<|gigatoken_236|>",
1909
+ "lstrip": false,
1910
+ "normalized": false,
1911
+ "rstrip": false,
1912
+ "single_word": false,
1913
+ "special": true
1914
+ },
1915
+ "128236": {
1916
+ "content": "<|gigatoken_237|>",
1917
+ "lstrip": false,
1918
+ "normalized": false,
1919
+ "rstrip": false,
1920
+ "single_word": false,
1921
+ "special": true
1922
+ },
1923
+ "128237": {
1924
+ "content": "<|gigatoken_238|>",
1925
+ "lstrip": false,
1926
+ "normalized": false,
1927
+ "rstrip": false,
1928
+ "single_word": false,
1929
+ "special": true
1930
+ },
1931
+ "128238": {
1932
+ "content": "<|gigatoken_239|>",
1933
+ "lstrip": false,
1934
+ "normalized": false,
1935
+ "rstrip": false,
1936
+ "single_word": false,
1937
+ "special": true
1938
+ },
1939
+ "128239": {
1940
+ "content": "<|gigatoken_240|>",
1941
+ "lstrip": false,
1942
+ "normalized": false,
1943
+ "rstrip": false,
1944
+ "single_word": false,
1945
+ "special": true
1946
+ },
1947
+ "128240": {
1948
+ "content": "<|gigatoken_241|>",
1949
+ "lstrip": false,
1950
+ "normalized": false,
1951
+ "rstrip": false,
1952
+ "single_word": false,
1953
+ "special": true
1954
+ },
1955
+ "128241": {
1956
+ "content": "<|gigatoken_242|>",
1957
+ "lstrip": false,
1958
+ "normalized": false,
1959
+ "rstrip": false,
1960
+ "single_word": false,
1961
+ "special": true
1962
+ },
1963
+ "128242": {
1964
+ "content": "<|gigatoken_243|>",
1965
+ "lstrip": false,
1966
+ "normalized": false,
1967
+ "rstrip": false,
1968
+ "single_word": false,
1969
+ "special": true
1970
+ },
1971
+ "128243": {
1972
+ "content": "<|gigatoken_244|>",
1973
+ "lstrip": false,
1974
+ "normalized": false,
1975
+ "rstrip": false,
1976
+ "single_word": false,
1977
+ "special": true
1978
+ },
1979
+ "128244": {
1980
+ "content": "<|gigatoken_245|>",
1981
+ "lstrip": false,
1982
+ "normalized": false,
1983
+ "rstrip": false,
1984
+ "single_word": false,
1985
+ "special": true
1986
+ },
1987
+ "128245": {
1988
+ "content": "<|gigatoken_246|>",
1989
+ "lstrip": false,
1990
+ "normalized": false,
1991
+ "rstrip": false,
1992
+ "single_word": false,
1993
+ "special": true
1994
+ },
1995
+ "128246": {
1996
+ "content": "<|gigatoken_247|>",
1997
+ "lstrip": false,
1998
+ "normalized": false,
1999
+ "rstrip": false,
2000
+ "single_word": false,
2001
+ "special": true
2002
+ },
2003
+ "128247": {
2004
+ "content": "<|gigatoken_248|>",
2005
+ "lstrip": false,
2006
+ "normalized": false,
2007
+ "rstrip": false,
2008
+ "single_word": false,
2009
+ "special": true
2010
+ },
2011
+ "128248": {
2012
+ "content": "<|gigatoken_249|>",
2013
+ "lstrip": false,
2014
+ "normalized": false,
2015
+ "rstrip": false,
2016
+ "single_word": false,
2017
+ "special": true
2018
+ },
2019
+ "128249": {
2020
+ "content": "<|gigatoken_250|>",
2021
+ "lstrip": false,
2022
+ "normalized": false,
2023
+ "rstrip": false,
2024
+ "single_word": false,
2025
+ "special": true
2026
+ },
2027
+ "128250": {
2028
+ "content": "<|gigatoken_251|>",
2029
+ "lstrip": false,
2030
+ "normalized": false,
2031
+ "rstrip": false,
2032
+ "single_word": false,
2033
+ "special": true
2034
+ },
2035
+ "128251": {
2036
+ "content": "<|gigatoken_252|>",
2037
+ "lstrip": false,
2038
+ "normalized": false,
2039
+ "rstrip": false,
2040
+ "single_word": false,
2041
+ "special": true
2042
+ },
2043
+ "128252": {
2044
+ "content": "<|gigatoken_253|>",
2045
+ "lstrip": false,
2046
+ "normalized": false,
2047
+ "rstrip": false,
2048
+ "single_word": false,
2049
+ "special": true
2050
+ },
2051
+ "128253": {
2052
+ "content": "<|gigatoken_254|>",
2053
+ "lstrip": false,
2054
+ "normalized": false,
2055
+ "rstrip": false,
2056
+ "single_word": false,
2057
+ "special": true
2058
+ },
2059
+ "128254": {
2060
+ "content": "<|gigatoken_255|>",
2061
+ "lstrip": false,
2062
+ "normalized": false,
2063
+ "rstrip": false,
2064
+ "single_word": false,
2065
+ "special": true
2066
+ },
2067
+ "128255": {
2068
+ "content": "<|gigatoken_256|>",
2069
+ "lstrip": false,
2070
+ "normalized": false,
2071
+ "rstrip": false,
2072
+ "single_word": false,
2073
+ "special": true
2074
+ }
2075
+ },
2076
+ "bos_token": "<s>",
2077
+ "clean_up_tokenization_spaces": true,
2078
+ "eos_token": "</s>",
2079
+ "extra_special_tokens": {},
2080
+ "legacy": true,
2081
+ "model_max_length": 1000000000000000019884624838656,
2082
+ "tokenizer_class": "PreTrainedTokenizerFast",
2083
+ "unk_token": "<unk>"
2084
+ }