Upload processor

Files changed (2) hide show

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -9261,7 +9261,7 @@
     "<and>"
   ],
   "bos_token": "<s>",
-  "clean_up_tokenization_spaces": true,
   "cls_token": "<s>",
   "eos_token": "</s>",
   "errors": "replace",

     "<and>"
   ],
   "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
   "cls_token": "<s>",
   "eos_token": "</s>",
   "errors": "replace",