Hunzla commited on
Commit
5bee8ed
1 Parent(s): 4a8adab

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +2 -3
  2. special_tokens_map.json +0 -7
  3. vocab.json +2 -2
added_tokens.json CHANGED
@@ -1,5 +1,4 @@
1
  {
2
- "</s>": 82,
3
- "<s>": 81,
4
- "[PAD]": 83
5
  }
 
1
  {
2
+ "</s>": 83,
3
+ "<s>": 82
 
4
  }
special_tokens_map.json CHANGED
@@ -13,13 +13,6 @@
13
  "normalized": true,
14
  "rstrip": false,
15
  "single_word": false
16
- },
17
- {
18
- "content": "[PAD]",
19
- "lstrip": false,
20
- "normalized": true,
21
- "rstrip": false,
22
- "single_word": false
23
  }
24
  ],
25
  "bos_token": "<s>",
 
13
  "normalized": true,
14
  "rstrip": false,
15
  "single_word": false
 
 
 
 
 
 
 
16
  }
17
  ],
18
  "bos_token": "<s>",
vocab.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  ",": 1,
3
- "[PAD]": 82,
4
- "[UNK]": 82,
5
  "|": 0,
6
  "ؑ": 2,
7
  "ؓ": 3,
 
1
  {
2
  ",": 1,
3
+ "[PAD]": 81,
4
+ "[UNK]": 80,
5
  "|": 0,
6
  "ؑ": 2,
7
  "ؓ": 3,