JustinLin610
commited on
Commit
•
cbaac2a
1
Parent(s):
3e2a9bf
Upload folder using huggingface_hub
Browse files- config.json +1 -1
- tokenizer.json +6 -10
config.json
CHANGED
@@ -26,4 +26,4 @@
|
|
26 |
"use_cache": true,
|
27 |
"use_sliding_window": false,
|
28 |
"vocab_size": 92416
|
29 |
-
}
|
|
|
26 |
"use_cache": true,
|
27 |
"use_sliding_window": false,
|
28 |
"vocab_size": 92416
|
29 |
+
}
|
tokenizer.json
CHANGED
@@ -12556,21 +12556,17 @@
|
|
12556 |
}
|
12557 |
]
|
12558 |
},
|
12559 |
-
"pre_tokenizer":
|
12560 |
-
"type": "Metaspace",
|
12561 |
-
"replacement": "▁",
|
12562 |
-
"add_prefix_space": false,
|
12563 |
-
"prepend_scheme": "first"
|
12564 |
-
},
|
12565 |
"post_processor": null,
|
12566 |
"decoder": {
|
12567 |
"type": "Sequence",
|
12568 |
"decoders": [
|
12569 |
{
|
12570 |
-
"type": "
|
12571 |
-
"
|
12572 |
-
|
12573 |
-
|
|
|
12574 |
},
|
12575 |
{
|
12576 |
"type": "ByteFallback"
|
|
|
12556 |
}
|
12557 |
]
|
12558 |
},
|
12559 |
+
"pre_tokenizer": null,
|
|
|
|
|
|
|
|
|
|
|
12560 |
"post_processor": null,
|
12561 |
"decoder": {
|
12562 |
"type": "Sequence",
|
12563 |
"decoders": [
|
12564 |
{
|
12565 |
+
"type": "Replace",
|
12566 |
+
"pattern": {
|
12567 |
+
"String": "▁"
|
12568 |
+
},
|
12569 |
+
"content": " "
|
12570 |
},
|
12571 |
{
|
12572 |
"type": "ByteFallback"
|