davda54 commited on
Commit
ee915d3
1 Parent(s): a16cb15

Upload tokenizer.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -3
tokenizer.json CHANGED
@@ -991,7 +991,6 @@
991
  {
992
  "type": "Metaspace",
993
  "replacement": "▁",
994
- "add_prefix_space": true,
995
  "prepend_scheme": "always"
996
  },
997
  {
@@ -1105,8 +1104,7 @@
1105
  {
1106
  "type": "Metaspace",
1107
  "replacement": "▁",
1108
- "add_prefix_space": false,
1109
- "prepend_scheme": "always"
1110
  },
1111
  {
1112
  "type": "Strip",
 
991
  {
992
  "type": "Metaspace",
993
  "replacement": "▁",
 
994
  "prepend_scheme": "always"
995
  },
996
  {
 
1104
  {
1105
  "type": "Metaspace",
1106
  "replacement": "▁",
1107
+ "prepend_scheme": "never"
 
1108
  },
1109
  {
1110
  "type": "Strip",