Update tokenizer.json
Browse filesSee https://github.com/huggingface/transformers/issues/28164
- tokenizer.json +1 -1
tokenizer.json
CHANGED
|
@@ -52,7 +52,7 @@
|
|
| 52 |
"normalizer": {
|
| 53 |
"type": "BertNormalizer",
|
| 54 |
"clean_text": true,
|
| 55 |
-
"handle_chinese_chars":
|
| 56 |
"strip_accents": null,
|
| 57 |
"lowercase": false
|
| 58 |
},
|
|
|
|
| 52 |
"normalizer": {
|
| 53 |
"type": "BertNormalizer",
|
| 54 |
"clean_text": true,
|
| 55 |
+
"handle_chinese_chars": true,
|
| 56 |
"strip_accents": null,
|
| 57 |
"lowercase": false
|
| 58 |
},
|