File size: 3,492 Bytes
4afdde5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
{
    "version": "1.0",
    "truncation": null,
    "padding": null,
    "added_tokens": [
        {
            "id": 73,
            "content": "<unk>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        }
    ],
    "normalizer": {
        "type": "Sequence",
        "normalizers": [
            {
                "type": "Lowercase"
            },
            {
                "type": "Replace",
                "pattern": {
                    "Regex": "[^\u0919\u0939\u0906\u092c\u0922\u09162\u0902\u0924\u091f_\u0938\u090a1\u0915\u0914\u0933\u091a\u0913\\-\u0908\u0936\u091c\u0943\u0917\u091d\u0288\u0910\u0926 \u0921\u090f\u0948\u094c97\u0920\u0931\u0935\u094b\u090b04\u092f6\u0903\u0932\u0927\u092a\u091e\u092b\u0909\u0928\u0937\u092e\u091b\u0947\u093f\u0923'\u0905\u0907\u093e\u0941\u0942\u0287\u0934\u0918\u094d\u0930\u092d\u0940\u0925]"
                },
                "content": ""
            },
            {
                "type": "Strip",
                "strip_left": true,
                "strip_right": true
            },
            {
                "type": "Replace",
                "pattern": {
                    "Regex": "(?=.)|(?<!^)$"
                },
                "content": "\u0919"
            }
        ]
    },
    "pre_tokenizer": {
        "type": "Split",
        "pattern": {
            "Regex": ""
        },
        "behavior": "Isolated",
        "invert": false
    },
    "post_processor": null,
    "decoder": null,
    "model": {
        "vocab": {
            "\u0919": 0,
            "\u0939": 1,
            "\u0906": 2,
            "\u092c": 3,
            "\u0922": 4,
            "\u0916": 5,
            "2": 6,
            "\u0902": 7,
            "\u0924": 8,
            "\u091f": 9,
            "_": 10,
            "\u0938": 11,
            "\u090a": 12,
            "1": 13,
            "\u0915": 14,
            "\u0914": 15,
            "\u0933": 16,
            "\u091a": 17,
            "\u0913": 18,
            "-": 19,
            "\u0908": 20,
            "\u0936": 21,
            "\u091c": 22,
            "\u0943": 23,
            "\u0917": 24,
            "\u091d": 25,
            "\u0288": 26,
            "\u0910": 27,
            "\u0926": 28,
            " ": 29,
            "\u0921": 30,
            "\u090f": 31,
            "\u0948": 32,
            "\u094c": 33,
            "9": 34,
            "7": 35,
            "\u0920": 36,
            "\u0931": 37,
            "\u0935": 38,
            "\u094b": 39,
            "\u090b": 40,
            "0": 41,
            "4": 42,
            "\u092f": 43,
            "6": 44,
            "\u0903": 45,
            "\u0932": 46,
            "\u0927": 47,
            "\u092a": 48,
            "\u091e": 49,
            "\u092b": 50,
            "\u0909": 51,
            "\u0928": 52,
            "\u0937": 53,
            "\u092e": 54,
            "\u091b": 55,
            "\u0947": 56,
            "\u093f": 57,
            "\u0923": 58,
            "'": 59,
            "\u0905": 60,
            "\u0907": 61,
            "\u093e": 62,
            "\u0941": 63,
            "\u0942": 64,
            "\u0287": 65,
            "\u0934": 66,
            "\u0918": 67,
            "\u094d": 68,
            "\u0930": 69,
            "\u092d": 70,
            "\u0940": 71,
            "\u0925": 72,
            "<unk>": 73
        }
    }
}