fixes to tokenizer

added_tokens.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f73effd45f282fdecbce3d5bda192b346d1e2e5dc024d4493ff276656001a5b6
+size 24

merges.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20832466756a988386123195ca6a4d1ecf92f0c1ff346872412fa54a8a2cb179
+size 546522

special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0b3c279b6ecdb71996a86ffb4d4ab94dfdb5df95f00bac9515688faef2ff5dd
+size 90

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:930138efc524dd7154f660fdfeafeac042f92640b7bf43df7c97fdc12e42140a
-size 1535805
+oid sha256:f5fbc311f419d79d43b03e4675c06f0006119fd550ed72f3159d411fc8f83a82
+size 1535928

tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85039a345c9cf46c6cdbb1789dd2b1885e0067c60ce8efa77c791f2aa467fa9b
+size 208

vocab.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba07a419f1e025bb93d757d738e3af7853128053df7b880d4fb7ab851b646207
+size 888217