Commit
·
b5a8819
1
Parent(s):
039ea66
restore
Browse files- tokenizer.json +7 -1
tokenizer.json
CHANGED
|
@@ -2085,7 +2085,13 @@
|
|
| 2085 |
"replacement": "▁",
|
| 2086 |
"add_prefix_space": true
|
| 2087 |
},
|
| 2088 |
-
"post_processor":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2089 |
"decoder": {
|
| 2090 |
"type": "Metaspace",
|
| 2091 |
"replacement": "▁",
|
|
|
|
| 2085 |
"replacement": "▁",
|
| 2086 |
"add_prefix_space": true
|
| 2087 |
},
|
| 2088 |
+
"post_processor": {
|
| 2089 |
+
"type": "RobertaProcessing",
|
| 2090 |
+
"sep": ["</s>", 1],
|
| 2091 |
+
"cls": ["<s>", 0],
|
| 2092 |
+
"trim_offsets": true,
|
| 2093 |
+
"add_prefix_space": false
|
| 2094 |
+
},
|
| 2095 |
"decoder": {
|
| 2096 |
"type": "Metaspace",
|
| 2097 |
"replacement": "▁",
|