imvladikon
commited on
Commit
·
97d18de
1
Parent(s):
1d2027f
Update tokenizer_config.json
Browse files- tokenizer_config.json +6 -0
tokenizer_config.json
CHANGED
@@ -110,6 +110,12 @@
|
|
110 |
"sp_model_kwargs": {},
|
111 |
"special_tokens_map_file": null,
|
112 |
"tokenizer_class": "EncT5Tokenizer",
|
|
|
|
|
|
|
|
|
|
|
|
|
113 |
"unk_token": "<unk>",
|
114 |
"use_fast": true
|
115 |
}
|
|
|
110 |
"sp_model_kwargs": {},
|
111 |
"special_tokens_map_file": null,
|
112 |
"tokenizer_class": "EncT5Tokenizer",
|
113 |
+
"auto_map": {
|
114 |
+
"AutoTokenizer": [
|
115 |
+
"configuration_enc_t5.EncT5Tokenizer",
|
116 |
+
null
|
117 |
+
]
|
118 |
+
}
|
119 |
"unk_token": "<unk>",
|
120 |
"use_fast": true
|
121 |
}
|