mwz commited on
Commit
8e79249
1 Parent(s): af90afc

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +82 -82
vocab.json CHANGED
@@ -1,88 +1,88 @@
1
  {
2
- "'": 65,
3
  "[PAD]": 85,
4
  "[UNK]": 84,
5
- "`": 49,
6
- "|": 46,
7
- "،": 83,
8
- "ؑ": 69,
9
- "ؓ": 48,
10
- "ؔ": 27,
11
- "؛": 7,
12
- "؟": 33,
13
- "ء": 29,
14
- "آ": 58,
15
- "أ": 0,
16
- "ؤ": 4,
17
- "ئ": 5,
18
- "ا": 71,
19
- "ب": 54,
20
- "ت": 35,
21
- "ث": 70,
22
- "ج": 8,
23
- "ح": 30,
24
- "خ": 18,
25
- "د": 14,
26
- "ذ": 53,
27
- "ر": 77,
28
- "ز": 81,
29
- "س": 32,
30
- "ش": 78,
31
- "ص": 68,
32
- "ض": 55,
33
- "ط": 15,
34
- "ظ": 43,
35
- "ع": 66,
36
- "غ": 25,
37
- "ف": 13,
38
- "ق": 28,
39
- "ك": 79,
40
- "ل": 82,
41
- "م": 6,
42
  "ن": 11,
43
- "ه": 50,
44
- "و": 36,
45
- "ى": 21,
46
- "ي": 3,
47
- "ً": 26,
48
- "َ": 16,
49
- "ُ": 73,
50
  "ِ": 31,
51
- "ّ": 57,
52
- "ْ": 52,
53
- "ٓ": 47,
54
- "ٔ": 37,
55
- "ٗ": 19,
56
- "ٰ": 59,
57
- "ٹ": 45,
58
- "پ": 74,
59
- "چ": 64,
60
- "ڈ": 38,
61
- "ڑ": 39,
62
- "ژ": 24,
63
- "ک": 2,
64
- "گ": 1,
65
- "ں": 40,
66
- "ھ": 17,
67
- "ہ": 20,
68
- "ۂ": 9,
69
- "ۃ": 10,
70
- "ی": 51,
71
- "ے": 60,
72
- "ۓ": 61,
73
- "۔": 42,
74
- "’": 63,
75
- "…": 67,
76
- "ﭨ": 75,
77
- "ﮭ": 41,
78
- "ﮯ": 44,
79
- "ﯾ": 56,
80
- "ﷲ": 12,
81
- "ﷺ": 34,
82
- "ﺗ": 76,
83
- "ﺘ": 22,
84
- "ﺩ": 80,
85
- "ﺲ": 72,
86
- "ﻧ": 23,
87
- "ﻮ": 62
88
  }
 
1
  {
2
+ "'": 10,
3
  "[PAD]": 85,
4
  "[UNK]": 84,
5
+ "`": 22,
6
+ "|": 1,
7
+ "،": 55,
8
+ "ؑ": 57,
9
+ "ؓ": 72,
10
+ "ؔ": 71,
11
+ "؛": 50,
12
+ "؟": 36,
13
+ "ء": 26,
14
+ "آ": 8,
15
+ "أ": 4,
16
+ "ؤ": 45,
17
+ "ئ": 48,
18
+ "ا": 39,
19
+ "ب": 12,
20
+ "ت": 67,
21
+ "ث": 83,
22
+ "ج": 78,
23
+ "ح": 66,
24
+ "خ": 74,
25
+ "د": 65,
26
+ "ذ": 5,
27
+ "ر": 82,
28
+ "ز": 60,
29
+ "س": 21,
30
+ "ش": 41,
31
+ "ص": 42,
32
+ "ض": 0,
33
+ "ط": 20,
34
+ "ظ": 49,
35
+ "ع": 17,
36
+ "غ": 81,
37
+ "ف": 70,
38
+ "ق": 46,
39
+ "ك": 6,
40
+ "ل": 28,
41
+ "م": 25,
42
  "ن": 11,
43
+ "ه": 34,
44
+ "و": 61,
45
+ "ى": 80,
46
+ "ي": 52,
47
+ "ً": 19,
48
+ "َ": 9,
49
+ "ُ": 15,
50
  "ِ": 31,
51
+ "ّ": 35,
52
+ "ْ": 27,
53
+ "ٓ": 2,
54
+ "ٔ": 23,
55
+ "ٗ": 47,
56
+ "ٰ": 16,
57
+ "ٹ": 38,
58
+ "پ": 69,
59
+ "چ": 40,
60
+ "ڈ": 54,
61
+ "ڑ": 33,
62
+ "ژ": 62,
63
+ "ک": 64,
64
+ "گ": 73,
65
+ "ں": 43,
66
+ "ھ": 75,
67
+ "ہ": 59,
68
+ "ۂ": 13,
69
+ "ۃ": 56,
70
+ "ی": 3,
71
+ "ے": 51,
72
+ "ۓ": 68,
73
+ "۔": 7,
74
+ "’": 79,
75
+ "…": 76,
76
+ "ﭨ": 53,
77
+ "ﮭ": 24,
78
+ "ﮯ": 63,
79
+ "ﯾ": 29,
80
+ "ﷲ": 14,
81
+ "ﷺ": 77,
82
+ "ﺗ": 32,
83
+ "ﺘ": 30,
84
+ "ﺩ": 37,
85
+ "ﺲ": 58,
86
+ "ﻧ": 44,
87
+ "ﻮ": 18
88
  }