Upload tokenizer (#3)
Browse files- Upload tokenizer (063c856bdba176a4341b6aedba759caa4669d5d3)
Co-authored-by: Full name <cat-state@users.noreply.huggingface.co>
- tokenizer.json +27 -0
tokenizer.json
CHANGED
@@ -227,6 +227,33 @@
|
|
227 |
"rstrip": false,
|
228 |
"normalized": true,
|
229 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
230 |
}
|
231 |
],
|
232 |
"normalizer": {
|
|
|
227 |
"rstrip": false,
|
228 |
"normalized": true,
|
229 |
"special": false
|
230 |
+
},
|
231 |
+
{
|
232 |
+
"id": 50277,
|
233 |
+
"content": "<|SYSTEM|>",
|
234 |
+
"single_word": false,
|
235 |
+
"lstrip": false,
|
236 |
+
"rstrip": false,
|
237 |
+
"normalized": false,
|
238 |
+
"special": true
|
239 |
+
},
|
240 |
+
{
|
241 |
+
"id": 50278,
|
242 |
+
"content": "<|USER|>",
|
243 |
+
"single_word": false,
|
244 |
+
"lstrip": false,
|
245 |
+
"rstrip": false,
|
246 |
+
"normalized": false,
|
247 |
+
"special": true
|
248 |
+
},
|
249 |
+
{
|
250 |
+
"id": 50279,
|
251 |
+
"content": "<|ASSISTANT|>",
|
252 |
+
"single_word": false,
|
253 |
+
"lstrip": false,
|
254 |
+
"rstrip": false,
|
255 |
+
"normalized": false,
|
256 |
+
"special": true
|
257 |
}
|
258 |
],
|
259 |
"normalizer": {
|