davda54 commited on
Commit
5675f08
1 Parent(s): eb4a743

update to the newer version of `tokenizers`

Browse files
Files changed (1) hide show
  1. tokenizer.json +2 -2
tokenizer.json CHANGED
@@ -991,7 +991,7 @@
991
  {
992
  "type": "Metaspace",
993
  "replacement": "▁",
994
- "add_prefix_space": true
995
  },
996
  {
997
  "type": "Split",
@@ -1104,7 +1104,7 @@
1104
  {
1105
  "type": "Metaspace",
1106
  "replacement": "▁",
1107
- "add_prefix_space": false
1108
  },
1109
  {
1110
  "type": "Strip",
 
991
  {
992
  "type": "Metaspace",
993
  "replacement": "▁",
994
+ "prepend_scheme": "always"
995
  },
996
  {
997
  "type": "Split",
 
1104
  {
1105
  "type": "Metaspace",
1106
  "replacement": "▁",
1107
+ "prepend_scheme": "always"
1108
  },
1109
  {
1110
  "type": "Strip",