Commit
·
8bde7dd
1
Parent(s):
13ad3a6
add tokenizer
Browse files- special_tokens_map.json +0 -1
- tokenizer_config.json +0 -1
- vocab.txt +0 -1
special_tokens_map.json
CHANGED
|
@@ -4,5 +4,4 @@
|
|
| 4 |
"pad_token": "[PAD]",
|
| 5 |
"sep_token": "[SEP]",
|
| 6 |
"unk_token": "[UNK]"
|
| 7 |
-
|
| 8 |
}
|
|
|
|
| 4 |
"pad_token": "[PAD]",
|
| 5 |
"sep_token": "[SEP]",
|
| 6 |
"unk_token": "[UNK]"
|
|
|
|
| 7 |
}
|
tokenizer_config.json
CHANGED
|
@@ -14,4 +14,3 @@
|
|
| 14 |
"truncation": true,
|
| 15 |
"unk_token": "[UNK]"
|
| 16 |
}
|
| 17 |
-
|
|
|
|
| 14 |
"truncation": true,
|
| 15 |
"unk_token": "[UNK]"
|
| 16 |
}
|
|
|
vocab.txt
CHANGED
|
@@ -28994,4 +28994,3 @@ subsp
|
|
| 28994 |
##－
|
| 28995 |
##／
|
| 28996 |
##：
|
| 28997 |
-
|
|
|
|
| 28994 |
##－
|
| 28995 |
##／
|
| 28996 |
##：
|
|
|