[WIP] Upload folder using huggingface_hub (multi-commit e84e4c0e1545628a53c87bb7530949d52709740c1280014e278a91726016ea53)

#1
README.md DELETED
@@ -1,23 +0,0 @@
1
- ---
2
- library_name: transformers
3
- tags:
4
- - mlx
5
- base_model: state-spaces/mamba-130m-hf
6
- ---
7
-
8
- # mlx-community/mamba-130m-hf-f32
9
-
10
- The Model [mlx-community/mamba-130m-hf-f32](https://huggingface.co/mlx-community/mamba-130m-hf-f32) was converted to MLX format from [state-spaces/mamba-130m-hf](https://huggingface.co/state-spaces/mamba-130m-hf) using mlx-lm version **0.18.1**.
11
-
12
- ## Use with mlx
13
-
14
- ```bash
15
- pip install mlx-lm
16
- ```
17
-
18
- ```python
19
- from mlx_lm import load, generate
20
-
21
- model, tokenizer = load("mlx-community/mamba-130m-hf-f32")
22
- response = generate(model, tokenizer, prompt="hello", verbose=True)
23
- ```
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
config.json DELETED
@@ -1,39 +0,0 @@
1
- {
2
- "architectures": [
3
- "MambaForCausalLM"
4
- ],
5
- "bos_token_id": 0,
6
- "conv_kernel": 4,
7
- "d_inner": 1536,
8
- "d_model": 768,
9
- "eos_token_id": 0,
10
- "expand": 2,
11
- "fused_add_norm": true,
12
- "hidden_act": "silu",
13
- "hidden_size": 768,
14
- "initializer_range": 0.1,
15
- "intermediate_size": 1536,
16
- "layer_norm_epsilon": 1e-05,
17
- "model_type": "mamba",
18
- "n_layer": 24,
19
- "num_hidden_layers": 24,
20
- "pad_token_id": 0,
21
- "pad_vocab_size_multiple": 8,
22
- "rescale_prenorm_residual": false,
23
- "residual_in_fp32": true,
24
- "rms_norm": true,
25
- "ssm_cfg": {},
26
- "state_size": 16,
27
- "time_step_floor": 0.0001,
28
- "time_step_init_scheme": "random",
29
- "time_step_max": 0.1,
30
- "time_step_min": 0.001,
31
- "time_step_rank": 48,
32
- "time_step_scale": 1.0,
33
- "torch_dtype": "float32",
34
- "transformers_version": "4.39.0.dev0",
35
- "use_bias": false,
36
- "use_cache": true,
37
- "use_conv_bias": true,
38
- "vocab_size": 50280
39
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:5b0c2400e3161bf9f12179770ff2c20db33e72f8678717a312e5eb2b2f341cbf
3
- size 516567397
 
 
 
 
model.safetensors.index.json DELETED
@@ -1,249 +0,0 @@
1
- {
2
- "metadata": {
3
- "total_size": 516541440
4
- },
5
- "weight_map": {
6
- "backbone.embeddings.weight": "model.safetensors",
7
- "backbone.layers.0.mixer.A_log": "model.safetensors",
8
- "backbone.layers.0.mixer.D": "model.safetensors",
9
- "backbone.layers.0.mixer.conv1d.bias": "model.safetensors",
10
- "backbone.layers.0.mixer.conv1d.weight": "model.safetensors",
11
- "backbone.layers.0.mixer.dt_proj.bias": "model.safetensors",
12
- "backbone.layers.0.mixer.dt_proj.weight": "model.safetensors",
13
- "backbone.layers.0.mixer.in_proj.weight": "model.safetensors",
14
- "backbone.layers.0.mixer.out_proj.weight": "model.safetensors",
15
- "backbone.layers.0.mixer.x_proj.weight": "model.safetensors",
16
- "backbone.layers.0.norm.weight": "model.safetensors",
17
- "backbone.layers.1.mixer.A_log": "model.safetensors",
18
- "backbone.layers.1.mixer.D": "model.safetensors",
19
- "backbone.layers.1.mixer.conv1d.bias": "model.safetensors",
20
- "backbone.layers.1.mixer.conv1d.weight": "model.safetensors",
21
- "backbone.layers.1.mixer.dt_proj.bias": "model.safetensors",
22
- "backbone.layers.1.mixer.dt_proj.weight": "model.safetensors",
23
- "backbone.layers.1.mixer.in_proj.weight": "model.safetensors",
24
- "backbone.layers.1.mixer.out_proj.weight": "model.safetensors",
25
- "backbone.layers.1.mixer.x_proj.weight": "model.safetensors",
26
- "backbone.layers.1.norm.weight": "model.safetensors",
27
- "backbone.layers.10.mixer.A_log": "model.safetensors",
28
- "backbone.layers.10.mixer.D": "model.safetensors",
29
- "backbone.layers.10.mixer.conv1d.bias": "model.safetensors",
30
- "backbone.layers.10.mixer.conv1d.weight": "model.safetensors",
31
- "backbone.layers.10.mixer.dt_proj.bias": "model.safetensors",
32
- "backbone.layers.10.mixer.dt_proj.weight": "model.safetensors",
33
- "backbone.layers.10.mixer.in_proj.weight": "model.safetensors",
34
- "backbone.layers.10.mixer.out_proj.weight": "model.safetensors",
35
- "backbone.layers.10.mixer.x_proj.weight": "model.safetensors",
36
- "backbone.layers.10.norm.weight": "model.safetensors",
37
- "backbone.layers.11.mixer.A_log": "model.safetensors",
38
- "backbone.layers.11.mixer.D": "model.safetensors",
39
- "backbone.layers.11.mixer.conv1d.bias": "model.safetensors",
40
- "backbone.layers.11.mixer.conv1d.weight": "model.safetensors",
41
- "backbone.layers.11.mixer.dt_proj.bias": "model.safetensors",
42
- "backbone.layers.11.mixer.dt_proj.weight": "model.safetensors",
43
- "backbone.layers.11.mixer.in_proj.weight": "model.safetensors",
44
- "backbone.layers.11.mixer.out_proj.weight": "model.safetensors",
45
- "backbone.layers.11.mixer.x_proj.weight": "model.safetensors",
46
- "backbone.layers.11.norm.weight": "model.safetensors",
47
- "backbone.layers.12.mixer.A_log": "model.safetensors",
48
- "backbone.layers.12.mixer.D": "model.safetensors",
49
- "backbone.layers.12.mixer.conv1d.bias": "model.safetensors",
50
- "backbone.layers.12.mixer.conv1d.weight": "model.safetensors",
51
- "backbone.layers.12.mixer.dt_proj.bias": "model.safetensors",
52
- "backbone.layers.12.mixer.dt_proj.weight": "model.safetensors",
53
- "backbone.layers.12.mixer.in_proj.weight": "model.safetensors",
54
- "backbone.layers.12.mixer.out_proj.weight": "model.safetensors",
55
- "backbone.layers.12.mixer.x_proj.weight": "model.safetensors",
56
- "backbone.layers.12.norm.weight": "model.safetensors",
57
- "backbone.layers.13.mixer.A_log": "model.safetensors",
58
- "backbone.layers.13.mixer.D": "model.safetensors",
59
- "backbone.layers.13.mixer.conv1d.bias": "model.safetensors",
60
- "backbone.layers.13.mixer.conv1d.weight": "model.safetensors",
61
- "backbone.layers.13.mixer.dt_proj.bias": "model.safetensors",
62
- "backbone.layers.13.mixer.dt_proj.weight": "model.safetensors",
63
- "backbone.layers.13.mixer.in_proj.weight": "model.safetensors",
64
- "backbone.layers.13.mixer.out_proj.weight": "model.safetensors",
65
- "backbone.layers.13.mixer.x_proj.weight": "model.safetensors",
66
- "backbone.layers.13.norm.weight": "model.safetensors",
67
- "backbone.layers.14.mixer.A_log": "model.safetensors",
68
- "backbone.layers.14.mixer.D": "model.safetensors",
69
- "backbone.layers.14.mixer.conv1d.bias": "model.safetensors",
70
- "backbone.layers.14.mixer.conv1d.weight": "model.safetensors",
71
- "backbone.layers.14.mixer.dt_proj.bias": "model.safetensors",
72
- "backbone.layers.14.mixer.dt_proj.weight": "model.safetensors",
73
- "backbone.layers.14.mixer.in_proj.weight": "model.safetensors",
74
- "backbone.layers.14.mixer.out_proj.weight": "model.safetensors",
75
- "backbone.layers.14.mixer.x_proj.weight": "model.safetensors",
76
- "backbone.layers.14.norm.weight": "model.safetensors",
77
- "backbone.layers.15.mixer.A_log": "model.safetensors",
78
- "backbone.layers.15.mixer.D": "model.safetensors",
79
- "backbone.layers.15.mixer.conv1d.bias": "model.safetensors",
80
- "backbone.layers.15.mixer.conv1d.weight": "model.safetensors",
81
- "backbone.layers.15.mixer.dt_proj.bias": "model.safetensors",
82
- "backbone.layers.15.mixer.dt_proj.weight": "model.safetensors",
83
- "backbone.layers.15.mixer.in_proj.weight": "model.safetensors",
84
- "backbone.layers.15.mixer.out_proj.weight": "model.safetensors",
85
- "backbone.layers.15.mixer.x_proj.weight": "model.safetensors",
86
- "backbone.layers.15.norm.weight": "model.safetensors",
87
- "backbone.layers.16.mixer.A_log": "model.safetensors",
88
- "backbone.layers.16.mixer.D": "model.safetensors",
89
- "backbone.layers.16.mixer.conv1d.bias": "model.safetensors",
90
- "backbone.layers.16.mixer.conv1d.weight": "model.safetensors",
91
- "backbone.layers.16.mixer.dt_proj.bias": "model.safetensors",
92
- "backbone.layers.16.mixer.dt_proj.weight": "model.safetensors",
93
- "backbone.layers.16.mixer.in_proj.weight": "model.safetensors",
94
- "backbone.layers.16.mixer.out_proj.weight": "model.safetensors",
95
- "backbone.layers.16.mixer.x_proj.weight": "model.safetensors",
96
- "backbone.layers.16.norm.weight": "model.safetensors",
97
- "backbone.layers.17.mixer.A_log": "model.safetensors",
98
- "backbone.layers.17.mixer.D": "model.safetensors",
99
- "backbone.layers.17.mixer.conv1d.bias": "model.safetensors",
100
- "backbone.layers.17.mixer.conv1d.weight": "model.safetensors",
101
- "backbone.layers.17.mixer.dt_proj.bias": "model.safetensors",
102
- "backbone.layers.17.mixer.dt_proj.weight": "model.safetensors",
103
- "backbone.layers.17.mixer.in_proj.weight": "model.safetensors",
104
- "backbone.layers.17.mixer.out_proj.weight": "model.safetensors",
105
- "backbone.layers.17.mixer.x_proj.weight": "model.safetensors",
106
- "backbone.layers.17.norm.weight": "model.safetensors",
107
- "backbone.layers.18.mixer.A_log": "model.safetensors",
108
- "backbone.layers.18.mixer.D": "model.safetensors",
109
- "backbone.layers.18.mixer.conv1d.bias": "model.safetensors",
110
- "backbone.layers.18.mixer.conv1d.weight": "model.safetensors",
111
- "backbone.layers.18.mixer.dt_proj.bias": "model.safetensors",
112
- "backbone.layers.18.mixer.dt_proj.weight": "model.safetensors",
113
- "backbone.layers.18.mixer.in_proj.weight": "model.safetensors",
114
- "backbone.layers.18.mixer.out_proj.weight": "model.safetensors",
115
- "backbone.layers.18.mixer.x_proj.weight": "model.safetensors",
116
- "backbone.layers.18.norm.weight": "model.safetensors",
117
- "backbone.layers.19.mixer.A_log": "model.safetensors",
118
- "backbone.layers.19.mixer.D": "model.safetensors",
119
- "backbone.layers.19.mixer.conv1d.bias": "model.safetensors",
120
- "backbone.layers.19.mixer.conv1d.weight": "model.safetensors",
121
- "backbone.layers.19.mixer.dt_proj.bias": "model.safetensors",
122
- "backbone.layers.19.mixer.dt_proj.weight": "model.safetensors",
123
- "backbone.layers.19.mixer.in_proj.weight": "model.safetensors",
124
- "backbone.layers.19.mixer.out_proj.weight": "model.safetensors",
125
- "backbone.layers.19.mixer.x_proj.weight": "model.safetensors",
126
- "backbone.layers.19.norm.weight": "model.safetensors",
127
- "backbone.layers.2.mixer.A_log": "model.safetensors",
128
- "backbone.layers.2.mixer.D": "model.safetensors",
129
- "backbone.layers.2.mixer.conv1d.bias": "model.safetensors",
130
- "backbone.layers.2.mixer.conv1d.weight": "model.safetensors",
131
- "backbone.layers.2.mixer.dt_proj.bias": "model.safetensors",
132
- "backbone.layers.2.mixer.dt_proj.weight": "model.safetensors",
133
- "backbone.layers.2.mixer.in_proj.weight": "model.safetensors",
134
- "backbone.layers.2.mixer.out_proj.weight": "model.safetensors",
135
- "backbone.layers.2.mixer.x_proj.weight": "model.safetensors",
136
- "backbone.layers.2.norm.weight": "model.safetensors",
137
- "backbone.layers.20.mixer.A_log": "model.safetensors",
138
- "backbone.layers.20.mixer.D": "model.safetensors",
139
- "backbone.layers.20.mixer.conv1d.bias": "model.safetensors",
140
- "backbone.layers.20.mixer.conv1d.weight": "model.safetensors",
141
- "backbone.layers.20.mixer.dt_proj.bias": "model.safetensors",
142
- "backbone.layers.20.mixer.dt_proj.weight": "model.safetensors",
143
- "backbone.layers.20.mixer.in_proj.weight": "model.safetensors",
144
- "backbone.layers.20.mixer.out_proj.weight": "model.safetensors",
145
- "backbone.layers.20.mixer.x_proj.weight": "model.safetensors",
146
- "backbone.layers.20.norm.weight": "model.safetensors",
147
- "backbone.layers.21.mixer.A_log": "model.safetensors",
148
- "backbone.layers.21.mixer.D": "model.safetensors",
149
- "backbone.layers.21.mixer.conv1d.bias": "model.safetensors",
150
- "backbone.layers.21.mixer.conv1d.weight": "model.safetensors",
151
- "backbone.layers.21.mixer.dt_proj.bias": "model.safetensors",
152
- "backbone.layers.21.mixer.dt_proj.weight": "model.safetensors",
153
- "backbone.layers.21.mixer.in_proj.weight": "model.safetensors",
154
- "backbone.layers.21.mixer.out_proj.weight": "model.safetensors",
155
- "backbone.layers.21.mixer.x_proj.weight": "model.safetensors",
156
- "backbone.layers.21.norm.weight": "model.safetensors",
157
- "backbone.layers.22.mixer.A_log": "model.safetensors",
158
- "backbone.layers.22.mixer.D": "model.safetensors",
159
- "backbone.layers.22.mixer.conv1d.bias": "model.safetensors",
160
- "backbone.layers.22.mixer.conv1d.weight": "model.safetensors",
161
- "backbone.layers.22.mixer.dt_proj.bias": "model.safetensors",
162
- "backbone.layers.22.mixer.dt_proj.weight": "model.safetensors",
163
- "backbone.layers.22.mixer.in_proj.weight": "model.safetensors",
164
- "backbone.layers.22.mixer.out_proj.weight": "model.safetensors",
165
- "backbone.layers.22.mixer.x_proj.weight": "model.safetensors",
166
- "backbone.layers.22.norm.weight": "model.safetensors",
167
- "backbone.layers.23.mixer.A_log": "model.safetensors",
168
- "backbone.layers.23.mixer.D": "model.safetensors",
169
- "backbone.layers.23.mixer.conv1d.bias": "model.safetensors",
170
- "backbone.layers.23.mixer.conv1d.weight": "model.safetensors",
171
- "backbone.layers.23.mixer.dt_proj.bias": "model.safetensors",
172
- "backbone.layers.23.mixer.dt_proj.weight": "model.safetensors",
173
- "backbone.layers.23.mixer.in_proj.weight": "model.safetensors",
174
- "backbone.layers.23.mixer.out_proj.weight": "model.safetensors",
175
- "backbone.layers.23.mixer.x_proj.weight": "model.safetensors",
176
- "backbone.layers.23.norm.weight": "model.safetensors",
177
- "backbone.layers.3.mixer.A_log": "model.safetensors",
178
- "backbone.layers.3.mixer.D": "model.safetensors",
179
- "backbone.layers.3.mixer.conv1d.bias": "model.safetensors",
180
- "backbone.layers.3.mixer.conv1d.weight": "model.safetensors",
181
- "backbone.layers.3.mixer.dt_proj.bias": "model.safetensors",
182
- "backbone.layers.3.mixer.dt_proj.weight": "model.safetensors",
183
- "backbone.layers.3.mixer.in_proj.weight": "model.safetensors",
184
- "backbone.layers.3.mixer.out_proj.weight": "model.safetensors",
185
- "backbone.layers.3.mixer.x_proj.weight": "model.safetensors",
186
- "backbone.layers.3.norm.weight": "model.safetensors",
187
- "backbone.layers.4.mixer.A_log": "model.safetensors",
188
- "backbone.layers.4.mixer.D": "model.safetensors",
189
- "backbone.layers.4.mixer.conv1d.bias": "model.safetensors",
190
- "backbone.layers.4.mixer.conv1d.weight": "model.safetensors",
191
- "backbone.layers.4.mixer.dt_proj.bias": "model.safetensors",
192
- "backbone.layers.4.mixer.dt_proj.weight": "model.safetensors",
193
- "backbone.layers.4.mixer.in_proj.weight": "model.safetensors",
194
- "backbone.layers.4.mixer.out_proj.weight": "model.safetensors",
195
- "backbone.layers.4.mixer.x_proj.weight": "model.safetensors",
196
- "backbone.layers.4.norm.weight": "model.safetensors",
197
- "backbone.layers.5.mixer.A_log": "model.safetensors",
198
- "backbone.layers.5.mixer.D": "model.safetensors",
199
- "backbone.layers.5.mixer.conv1d.bias": "model.safetensors",
200
- "backbone.layers.5.mixer.conv1d.weight": "model.safetensors",
201
- "backbone.layers.5.mixer.dt_proj.bias": "model.safetensors",
202
- "backbone.layers.5.mixer.dt_proj.weight": "model.safetensors",
203
- "backbone.layers.5.mixer.in_proj.weight": "model.safetensors",
204
- "backbone.layers.5.mixer.out_proj.weight": "model.safetensors",
205
- "backbone.layers.5.mixer.x_proj.weight": "model.safetensors",
206
- "backbone.layers.5.norm.weight": "model.safetensors",
207
- "backbone.layers.6.mixer.A_log": "model.safetensors",
208
- "backbone.layers.6.mixer.D": "model.safetensors",
209
- "backbone.layers.6.mixer.conv1d.bias": "model.safetensors",
210
- "backbone.layers.6.mixer.conv1d.weight": "model.safetensors",
211
- "backbone.layers.6.mixer.dt_proj.bias": "model.safetensors",
212
- "backbone.layers.6.mixer.dt_proj.weight": "model.safetensors",
213
- "backbone.layers.6.mixer.in_proj.weight": "model.safetensors",
214
- "backbone.layers.6.mixer.out_proj.weight": "model.safetensors",
215
- "backbone.layers.6.mixer.x_proj.weight": "model.safetensors",
216
- "backbone.layers.6.norm.weight": "model.safetensors",
217
- "backbone.layers.7.mixer.A_log": "model.safetensors",
218
- "backbone.layers.7.mixer.D": "model.safetensors",
219
- "backbone.layers.7.mixer.conv1d.bias": "model.safetensors",
220
- "backbone.layers.7.mixer.conv1d.weight": "model.safetensors",
221
- "backbone.layers.7.mixer.dt_proj.bias": "model.safetensors",
222
- "backbone.layers.7.mixer.dt_proj.weight": "model.safetensors",
223
- "backbone.layers.7.mixer.in_proj.weight": "model.safetensors",
224
- "backbone.layers.7.mixer.out_proj.weight": "model.safetensors",
225
- "backbone.layers.7.mixer.x_proj.weight": "model.safetensors",
226
- "backbone.layers.7.norm.weight": "model.safetensors",
227
- "backbone.layers.8.mixer.A_log": "model.safetensors",
228
- "backbone.layers.8.mixer.D": "model.safetensors",
229
- "backbone.layers.8.mixer.conv1d.bias": "model.safetensors",
230
- "backbone.layers.8.mixer.conv1d.weight": "model.safetensors",
231
- "backbone.layers.8.mixer.dt_proj.bias": "model.safetensors",
232
- "backbone.layers.8.mixer.dt_proj.weight": "model.safetensors",
233
- "backbone.layers.8.mixer.in_proj.weight": "model.safetensors",
234
- "backbone.layers.8.mixer.out_proj.weight": "model.safetensors",
235
- "backbone.layers.8.mixer.x_proj.weight": "model.safetensors",
236
- "backbone.layers.8.norm.weight": "model.safetensors",
237
- "backbone.layers.9.mixer.A_log": "model.safetensors",
238
- "backbone.layers.9.mixer.D": "model.safetensors",
239
- "backbone.layers.9.mixer.conv1d.bias": "model.safetensors",
240
- "backbone.layers.9.mixer.conv1d.weight": "model.safetensors",
241
- "backbone.layers.9.mixer.dt_proj.bias": "model.safetensors",
242
- "backbone.layers.9.mixer.dt_proj.weight": "model.safetensors",
243
- "backbone.layers.9.mixer.in_proj.weight": "model.safetensors",
244
- "backbone.layers.9.mixer.out_proj.weight": "model.safetensors",
245
- "backbone.layers.9.mixer.x_proj.weight": "model.safetensors",
246
- "backbone.layers.9.norm.weight": "model.safetensors",
247
- "backbone.norm_f.weight": "model.safetensors"
248
- }
249
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
special_tokens_map.json DELETED
@@ -1,30 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<|endoftext|>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "<|endoftext|>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "pad_token": {
17
- "content": "<|endoftext|>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- },
23
- "unk_token": {
24
- "content": "<|endoftext|>",
25
- "lstrip": false,
26
- "normalized": false,
27
- "rstrip": false,
28
- "single_word": false
29
- }
30
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tokenizer.json DELETED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json DELETED
@@ -1,218 +0,0 @@
1
- {
2
- "add_bos_token": false,
3
- "add_eos_token": false,
4
- "add_prefix_space": false,
5
- "added_tokens_decoder": {
6
- "0": {
7
- "content": "<|endoftext|>",
8
- "lstrip": false,
9
- "normalized": false,
10
- "rstrip": false,
11
- "single_word": false,
12
- "special": true
13
- },
14
- "1": {
15
- "content": "<|padding|>",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
21
- },
22
- "50254": {
23
- "content": " ",
24
- "lstrip": false,
25
- "normalized": true,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": false
29
- },
30
- "50255": {
31
- "content": " ",
32
- "lstrip": false,
33
- "normalized": true,
34
- "rstrip": false,
35
- "single_word": false,
36
- "special": false
37
- },
38
- "50256": {
39
- "content": " ",
40
- "lstrip": false,
41
- "normalized": true,
42
- "rstrip": false,
43
- "single_word": false,
44
- "special": false
45
- },
46
- "50257": {
47
- "content": " ",
48
- "lstrip": false,
49
- "normalized": true,
50
- "rstrip": false,
51
- "single_word": false,
52
- "special": false
53
- },
54
- "50258": {
55
- "content": " ",
56
- "lstrip": false,
57
- "normalized": true,
58
- "rstrip": false,
59
- "single_word": false,
60
- "special": false
61
- },
62
- "50259": {
63
- "content": " ",
64
- "lstrip": false,
65
- "normalized": true,
66
- "rstrip": false,
67
- "single_word": false,
68
- "special": false
69
- },
70
- "50260": {
71
- "content": " ",
72
- "lstrip": false,
73
- "normalized": true,
74
- "rstrip": false,
75
- "single_word": false,
76
- "special": false
77
- },
78
- "50261": {
79
- "content": " ",
80
- "lstrip": false,
81
- "normalized": true,
82
- "rstrip": false,
83
- "single_word": false,
84
- "special": false
85
- },
86
- "50262": {
87
- "content": " ",
88
- "lstrip": false,
89
- "normalized": true,
90
- "rstrip": false,
91
- "single_word": false,
92
- "special": false
93
- },
94
- "50263": {
95
- "content": " ",
96
- "lstrip": false,
97
- "normalized": true,
98
- "rstrip": false,
99
- "single_word": false,
100
- "special": false
101
- },
102
- "50264": {
103
- "content": " ",
104
- "lstrip": false,
105
- "normalized": true,
106
- "rstrip": false,
107
- "single_word": false,
108
- "special": false
109
- },
110
- "50265": {
111
- "content": " ",
112
- "lstrip": false,
113
- "normalized": true,
114
- "rstrip": false,
115
- "single_word": false,
116
- "special": false
117
- },
118
- "50266": {
119
- "content": " ",
120
- "lstrip": false,
121
- "normalized": true,
122
- "rstrip": false,
123
- "single_word": false,
124
- "special": false
125
- },
126
- "50267": {
127
- "content": " ",
128
- "lstrip": false,
129
- "normalized": true,
130
- "rstrip": false,
131
- "single_word": false,
132
- "special": false
133
- },
134
- "50268": {
135
- "content": " ",
136
- "lstrip": false,
137
- "normalized": true,
138
- "rstrip": false,
139
- "single_word": false,
140
- "special": false
141
- },
142
- "50269": {
143
- "content": " ",
144
- "lstrip": false,
145
- "normalized": true,
146
- "rstrip": false,
147
- "single_word": false,
148
- "special": false
149
- },
150
- "50270": {
151
- "content": " ",
152
- "lstrip": false,
153
- "normalized": true,
154
- "rstrip": false,
155
- "single_word": false,
156
- "special": false
157
- },
158
- "50271": {
159
- "content": " ",
160
- "lstrip": false,
161
- "normalized": true,
162
- "rstrip": false,
163
- "single_word": false,
164
- "special": false
165
- },
166
- "50272": {
167
- "content": " ",
168
- "lstrip": false,
169
- "normalized": true,
170
- "rstrip": false,
171
- "single_word": false,
172
- "special": false
173
- },
174
- "50273": {
175
- "content": " ",
176
- "lstrip": false,
177
- "normalized": true,
178
- "rstrip": false,
179
- "single_word": false,
180
- "special": false
181
- },
182
- "50274": {
183
- "content": " ",
184
- "lstrip": false,
185
- "normalized": true,
186
- "rstrip": false,
187
- "single_word": false,
188
- "special": false
189
- },
190
- "50275": {
191
- "content": " ",
192
- "lstrip": false,
193
- "normalized": true,
194
- "rstrip": false,
195
- "single_word": false,
196
- "special": false
197
- },
198
- "50276": {
199
- "content": " ",
200
- "lstrip": false,
201
- "normalized": true,
202
- "rstrip": false,
203
- "single_word": false,
204
- "special": false
205
- }
206
- },
207
- "bos_token": "<|endoftext|>",
208
- "clean_up_tokenization_spaces": true,
209
- "eos_token": "<|endoftext|>",
210
- "max_length": 1024,
211
- "model_max_length": 1000000000000000019884624838656,
212
- "pad_token": "<|endoftext|>",
213
- "stride": 0,
214
- "tokenizer_class": "GPTNeoXTokenizer",
215
- "truncation_side": "right",
216
- "truncation_strategy": "longest_first",
217
- "unk_token": "<|endoftext|>"
218
- }