Trkkk commited on
Commit
dcf0670
·
verified ·
1 Parent(s): d719d45

Upload processor

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -44,6 +44,7 @@
44
  "clean_up_tokenization_spaces": false,
45
  "cls_token": "[CLS]",
46
  "do_lower_case": true,
 
47
  "mask_token": "[MASK]",
48
  "model_input_names": [
49
  "input_ids",
 
44
  "clean_up_tokenization_spaces": false,
45
  "cls_token": "[CLS]",
46
  "do_lower_case": true,
47
+ "extra_special_tokens": {},
48
  "mask_token": "[MASK]",
49
  "model_input_names": [
50
  "input_ids",