dongxiaoqun
commited on
Commit
•
a9ae18c
1
Parent(s):
fc31f0d
Upload tokenizer_config.json
Browse files- tokenizer_config.json +1 -0
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"do_lower_case": true, "do_basic_tokenize": true, "never_split": null, "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": "<mask_2>", "eos_token": "</s>", "tokenize_chinese_chars": true, "additional_special_tokens": ["<mask_1>"], "strip_accents": null, "special_tokens_map_file": "/home/dongxiaoqun/.cache/huggingface/transformers/9e23309b8a9cf9869349434d8c45d1552726a48b7533e026e07681b3f86af880.28e3ba665aa5ffe2c43b3caab35c07f9ebb4a77fb1aa5ff611df4edd8eb90653", "tokenizer_file": null, "name_or_path": "IDEA-CCNL/Randeng-Pegasus-238M-Summary-Chinese", "tokenizer_class": "PegasusTokenizer"}
|