Upload tokenizer.json (#28)
Browse files- Upload tokenizer.json (d749371bfe914cc1a3a9e6bee38a38825a3c697b)
- tokenizer.json +1 -1
tokenizer.json
CHANGED
|
@@ -203,7 +203,7 @@
|
|
| 203 |
},
|
| 204 |
{
|
| 205 |
"id": 50279,
|
| 206 |
-
"content": "<|
|
| 207 |
"single_word": false,
|
| 208 |
"lstrip": false,
|
| 209 |
"rstrip": false,
|
|
|
|
| 203 |
},
|
| 204 |
{
|
| 205 |
"id": 50279,
|
| 206 |
+
"content": "<|he|>",
|
| 207 |
"single_word": false,
|
| 208 |
"lstrip": false,
|
| 209 |
"rstrip": false,
|