sdadas commited on
Commit
bea9c6c
1 Parent(s): 7f52af6

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -1
tokenizer.json CHANGED
@@ -67,7 +67,7 @@
67
  "pre_tokenizer": {
68
  "type": "Metaspace",
69
  "replacement": "▁",
70
- "add_prefix_space": false,
71
  "prepend_scheme": "always"
72
  },
73
  "post_processor": {
 
67
  "pre_tokenizer": {
68
  "type": "Metaspace",
69
  "replacement": "▁",
70
+ "add_prefix_space": true,
71
  "prepend_scheme": "always"
72
  },
73
  "post_processor": {