Upload tokenizer

merges.txt

@@ -1,444 +1 @@
 #version: 0.2
-< |
-| >
-e p
-s ep
-ا ل
-و َ
-b sep
-v sep
-أ َ
-م َ
-ع َ
-ل َ
-ت َ
-م ِ
-ن َ
-ّ َ
-ب ِ
-م ا
-ي َ
-ر َ
-م ُ
-ق َ
-ه ُ
-ك َ
-ف َ
-ب َ
-إ ِ
-ه ا
-ر ِ
-ل ِ
-ل ا
-ن ا
-و ا
-د َ
-ه ِ
-مِ ن
-ف ي
-ح َ
-ر ا
-ن ِ
-ل ى
-س َ
-ت ُ
-ج َ
-ر ُ
-د ِ
-ع ِ
-ع ُ
-ي ا
-غ َ
-ك ُ
-أَ ن
-د ا
-ل ُ
-ّ ِ
-ب ا
-ال أَ
-ذ ا
-ط َ
-ّ ا
-p sep
-ّ ُ
-ن ي
-ال مَ
-ع ا
-ه َ
-د ُ
-ش َ
-ب ُ
-ي ُ
-عَ لى
-ق ِ
-وَ ال
-ن ُ
-ق ا
-لَ ي
-إِ ن
-د ي
-ق ُ
-خ َ
-ئ ِ
-ا ِ
-ض َ
-ص َ
-ل ي
-بِ ال
-ر ي
-ز َ
-ت ِ
-ب ي
-إِ ذا
-س ِ
-ح ِ
-ف ِ
-س ا
-قَ د
-ك ا
-عَ ن
-لَ م
-ج ِ
-وَ أَ
-ح ُ
-ج ا
-ل َّ
-ف ا
-ال ِ
-م ي
-أ ُ
-مَ ن
-e r
-e t
-m et
-er _
-met er_
-مِ نَ
-ي نَ
-ف ُ
-ال مُ
-ال عَ
-ة ٍ
-وَ لا
-ط ا
-ح ا
-س ُ
-لِ ل
-أَن َّ
-ء ِ
-ذ َ
-غَ ي
-ال قَ
-ة ً
-إِ لى
-أَ و
-ال نَ
-ك ِ
-ط ِ
-ر ى
-ض ِ
-بَ ع
-ء َ
-ل و
-يَ و
-كُ ل
-ث َ
-ر و
-د و
-ر َّ
-بِ هِ
-س تَ
-لَ هُ
-ق ي
-ص ا
-لَ و
-ال ي
-ظ َ
-ل ّا
-ج ُ
-ال رَ
-ّ ي
-ع ي
-ال َ
-ز ِ
-ض ا
-ة ٌ
-ن َّ
-غ ُ
-ّ ى
-ش ا
-غ ا
-و ى
-ة ِ
-د ى
-هُ م
-أَ ع
-م َّ
-ص ِ
-ذ ي
-إِن َّ
-ال عُ
-ما ً
-ال حَ
-كَ أَنَّ
-ز ا
-إِ لّا
-ال سَ
-وَ ما
-ش ِ
-كا نَ
-أَ ي
-ال جَ
-قَ ل
-م ٍ
-ء ُ
-خ ُ
-حَ ت
-بَ ينَ
-ال دَ
-ر ٍ
-د ٍ
-ض ُ
-ج و
-ذ ُ
-ي َّ
-م و
-دا ً
-ال شَ
-زَ ما
-ة َ
-را ً
-ت ي
-ذ ِ
-بَع دَ
-د َّ
-ق و
-وَ يَ
-وَ تَ
-ظ ُ
-ش ُ
-ي و
-بِ ها
-ال بَ
-ظ ِ
-ال غَ
-ض ي
-ال عِ
-كُ ن
-ال وَ
-س ي
-و ِ
-ب َّ
-وَ اِ
-ص ُ
-عَ لَي
-كُل ِّ
-أَ ر
-ت ا
-ث ُ
-خ ِ
-ال خَ
-ب ٍ
-با ً
-خ ا
-غ ِ
-ال َّ
-ال فَ
-وَ إِن
-ر ٌ
-ط ُ
-وَ قَد
-في هِ
-حَت ّى
-وَ لَ
-ال رِ
-عِ ن
-ي نِ
-ال ُ
-أَ ب
-ن و
-كُ م
-ط و
-ل ٍ
-ال لَي
-أَ م
-ة ُ
-لا ً
-ال هَ
-ج ي
-ن ى
-ال زَما
-كَ م
-م ٌ
-مِن هُ
-ال طَ
-ب ٌ
-ال صَ
-ال نا
-م ى
-لَ ها
-ط ي
-ع و
-ي ّا
-ق ى
-إِ لَي
-يَ ن
-و ُ
-الدَ ه
-وَ مَ
-و َّ
-يَو مَ
-الَّ ذي
-المَ ج
-ر ُّ
-كُ لَّ
-ي ِّ
-ف و
-عا ً
-نا ً
-ب و
-س و
-ال ا
-أَ ح
-تُ هُ
-وا رِ
-ح ي
-ال دُ
-meter_ 0
-ل ٌ
-قَ و
-قَ ب
-وَ ج
-د ُّ
-مَ ع
-د ٌ
-أَن تَ
-ال حُ
-العُ لى
-ل ُّ
-meter_ 3
-ال تَ
-عَ ي
-عَ نِ
-ت ى
-ث ا
-وَ لَم
-غَي رِ
-وَ مُ
-ق ٍ
-عِن دَ
-ب ى
-رَ ت
-ض ى
-تَ ح
-وا ئِ
-ز ُ
-أَ ق
-بَ ي
-اِ ن
-هُ مُ
-أَ س
-فَ ما
-ث ِ
-كَ ما
-كُل ُّ
-ل ِّ
-وَ نَ
-أَ ط
-وَ رَ
-مَ و
-هِ م
-د ِّ
-ر ِّ
-ن ٍ
-ال إِ
-تَ هُ
-لَي سَ
-وَ لَو
-غَي رَ
-ّ ٍ
-مِ ث
-وَ ه
-نَ ف
-الأَ ر
-ال قُ
-مُ ن
-دو نَ
-ال حِ
-الأَ ع
-كَ ي
-عَ لي
-ع ٌ
-ع ٍ
-ال لَ
-ما ءِ
-أَ ل
-تِ هِ
-ال نُ
-ال كَ
-عَ ز
-كُن تُ
-ش ي
-أَ نا
-حَ ي
-ذا كَ
-يَ ع
-هَ ل
-ص و
-ال ضَ
-لَ كَ
-أَ خ
-مُ تَ
-وَ ق
-مَ ر
-يَ ر
-وَ جَ
-ّ ٌ
-ال أُ
-في ها
-رَ أ
-قُ ل
-فَ لا
-فَ اِ
-القَ نا
-مَ ج
-هَ ذا
-قا ً
-قَل بي
-ال سُ
-عَ ت
-سَ ي
-ف َّ
-ال جِ
-ال خُ
-غَي رُ
-ظ ا
-ه و
-ال رُ
-دَ م
-كُ مُ
-ك و
-وَ عَ
-المَ ع
-اللَي الي
-ن ٌ
-غ ي
-أَ ج
-م ِّ
-م ُّ
-وَ قَ
-ال ظَ
-تَ ر
-تَ ن
-دَ ت
-ل تَ
-ال آ
-طَ و
-ال بُ
-يَو ماً
-وَ مِن
-وَ يُ
-ز ي
-مَ ط
-تَ رَ
-خَ ي

tokenizer.json

@@ -173,11 +173,65 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 19,
+      "content": "<|meter_14|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 20,
+      "content": "<|meter_15|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 21,
+      "content": "<|meter_16|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 22,
+      "content": "<|meter_17|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 23,
+      "content": "<|meter_18|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 24,
+      "content": "<|meter_19|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
   "pre_tokenizer": {
-    "type": "Whitespace"
+    "type": "UnicodeScripts"
   },
   "post_processor": null,
   "decoder": null,
@@ -209,961 +263,81 @@
       "<|meter_11|>": 16,
       "<|meter_12|>": 17,
       "<|meter_13|>": 18,
-      "0": 19,
-      "1": 20,
-      "2": 21,
-      "3": 22,
-      "4": 23,
-      "5": 24,
-      "6": 25,
-      "7": 26,
-      "8": 27,
-      "9": 28,
-      ":": 29,
-      "<": 30,
-      ">": 31,
-      "_": 32,
-      "b": 33,
-      "e": 34,
-      "m": 35,
-      "p": 36,
-      "r": 37,
-      "s": 38,
-      "t": 39,
-      "v": 40,
-      "|": 41,
-      "ء": 42,
-      "آ": 43,
-      "أ": 44,
-      "ؤ": 45,
-      "إ": 46,
-      "ئ": 47,
-      "ا": 48,
-      "ب": 49,
-      "ة": 50,
-      "ت": 51,
-      "ث": 52,
-      "ج": 53,
-      "ح": 54,
-      "خ": 55,
-      "د": 56,
-      "ذ": 57,
-      "ر": 58,
-      "ز": 59,
-      "س": 60,
-      "ش": 61,
-      "ص": 62,
-      "ض": 63,
-      "ط": 64,
-      "ظ": 65,
-      "ع": 66,
-      "غ": 67,
-      "ف": 68,
-      "ق": 69,
-      "ك": 70,
-      "ل": 71,
-      "م": 72,
-      "ن": 73,
-      "ه": 74,
-      "و": 75,
-      "ى": 76,
-      "ي": 77,
-      "ً": 78,
-      "ٌ": 79,
-      "ٍ": 80,
-      "َ": 81,
-      "ُ": 82,
-      "ِ": 83,
-      "ّ": 84,
-      "ْ": 85,
-      "<|": 86,
-      "|>": 87,
-      "ep": 88,
-      "sep": 89,
-      "ال": 90,
-      "وَ": 91,
-      "bsep": 92,
-      "vsep": 93,
-      "أَ": 94,
-      "مَ": 95,
-      "عَ": 96,
-      "لَ": 97,
-      "تَ": 98,
-      "مِ": 99,
-      "نَ": 100,
-      "َّ": 101,
-      "بِ": 102,
-      "ما": 103,
-      "يَ": 104,
-      "رَ": 105,
-      "مُ": 106,
-      "قَ": 107,
-      "هُ": 108,
-      "كَ": 109,
-      "فَ": 110,
-      "بَ": 111,
-      "إِ": 112,
-      "ها": 113,
-      "رِ": 114,
-      "لِ": 115,
-      "لا": 116,
-      "نا": 117,
-      "وا": 118,
-      "دَ": 119,
-      "هِ": 120,
-      "مِن": 121,
-      "في": 122,
-      "حَ": 123,
-      "را": 124,
-      "نِ": 125,
-      "لى": 126,
-      "سَ": 127,
-      "تُ": 128,
-      "جَ": 129,
-      "رُ": 130,
-      "دِ": 131,
-      "عِ": 132,
-      "عُ": 133,
-      "يا": 134,
-      "غَ": 135,
-      "كُ": 136,
-      "أَن": 137,
-      "دا": 138,
-      "لُ": 139,
-      "ِّ": 140,
-      "با": 141,
-      "الأَ": 142,
-      "ذا": 143,
-      "طَ": 144,
-      "ّا": 145,
-      "psep": 146,
-      "ُّ": 147,
-      "ني": 148,
-      "المَ": 149,
-      "عا": 150,
-      "هَ": 151,
-      "دُ": 152,
-      "شَ": 153,
-      "بُ": 154,
-      "يُ": 155,
-      "عَلى": 156,
-      "قِ": 157,
-      "وَال": 158,
-      "نُ": 159,
-      "قا": 160,
-      "لَي": 161,
-      "إِن": 162,
-      "دي": 163,
-      "قُ": 164,
-      "خَ": 165,
-      "ئِ": 166,
-      "اِ": 167,
-      "ضَ": 168,
-      "صَ": 169,
-      "لي": 170,
-      "بِال": 171,
-      "ري": 172,
-      "زَ": 173,
-      "تِ": 174,
-      "بي": 175,
-      "إِذا": 176,
-      "سِ": 177,
-      "حِ": 178,
-      "فِ": 179,
-      "سا": 180,
-      "قَد": 181,
-      "كا": 182,
-      "عَن": 183,
-      "لَم": 184,
-      "جِ": 185,
-      "وَأَ": 186,
-      "حُ": 187,
-      "جا": 188,
-      "لَّ": 189,
-      "فا": 190,
-      "الِ": 191,
-      "مي": 192,
-      "أُ": 193,
-      "مَن": 194,
-      "er": 195,
-      "et": 196,
-      "met": 197,
-      "er_": 198,
-      "meter_": 199,
-      "مِنَ": 200,
-      "ينَ": 201,
-      "فُ": 202,
-      "المُ": 203,
-      "العَ": 204,
-      "ةٍ": 205,
-      "وَلا": 206,
-      "ط��": 207,
-      "حا": 208,
-      "سُ": 209,
-      "لِل": 210,
-      "أَنَّ": 211,
-      "ءِ": 212,
-      "ذَ": 213,
-      "غَي": 214,
-      "القَ": 215,
-      "ةً": 216,
-      "إِلى": 217,
-      "أَو": 218,
-      "النَ": 219,
-      "كِ": 220,
-      "طِ": 221,
-      "رى": 222,
-      "ضِ": 223,
-      "بَع": 224,
-      "ءَ": 225,
-      "لو": 226,
-      "يَو": 227,
-      "كُل": 228,
-      "ثَ": 229,
-      "رو": 230,
-      "دو": 231,
-      "رَّ": 232,
-      "بِهِ": 233,
-      "ستَ": 234,
-      "لَهُ": 235,
-      "قي": 236,
-      "صا": 237,
-      "لَو": 238,
-      "الي": 239,
-      "ظَ": 240,
-      "لّا": 241,
-      "جُ": 242,
-      "الرَ": 243,
-      "ّي": 244,
-      "عي": 245,
-      "الَ": 246,
-      "زِ": 247,
-      "ضا": 248,
-      "ةٌ": 249,
-      "نَّ": 250,
-      "غُ": 251,
-      "ّى": 252,
-      "شا": 253,
-      "غا": 254,
-      "وى": 255,
-      "ةِ": 256,
-      "دى": 257,
-      "هُم": 258,
-      "أَع": 259,
-      "مَّ": 260,
-      "صِ": 261,
-      "ذي": 262,
-      "إِنَّ": 263,
-      "العُ": 264,
-      "ماً": 265,
-      "الحَ": 266,
-      "كَأَنَّ": 267,
-      "زا": 268,
-      "إِلّا": 269,
-      "السَ": 270,
-      "وَما": 271,
-      "شِ": 272,
-      "كانَ": 273,
-      "أَي": 274,
-      "الجَ": 275,
-      "قَل": 276,
-      "مٍ": 277,
-      "ءُ": 278,
-      "خُ": 279,
-      "حَت": 280,
-      "بَينَ": 281,
-      "الدَ": 282,
-      "رٍ": 283,
-      "دٍ": 284,
-      "ضُ": 285,
-      "جو": 286,
-      "ذُ": 287,
-      "يَّ": 288,
-      "مو": 289,
-      "داً": 290,
-      "الشَ": 291,
-      "زَما": 292,
-      "ةَ": 293,
-      "راً": 294,
-      "تي": 295,
-      "ذِ": 296,
-      "بَعدَ": 297,
-      "دَّ": 298,
-      "قو": 299,
-      "وَيَ": 300,
-      "وَتَ": 301,
-      "ظُ": 302,
-      "شُ": 303,
-      "يو": 304,
-      "بِها": 305,
-      "البَ": 306,
-      "ظِ": 307,
-      "الغَ": 308,
-      "ضي": 309,
-      "العِ": 310,
-      "كُن": 311,
-      "الوَ": 312,
-      "سي": 313,
-      "وِ": 314,
-      "بَّ": 315,
-      "وَاِ": 316,
-      "صُ": 317,
-      "عَلَي": 318,
-      "كُلِّ": 319,
-      "أَر": 320,
-      "تا": 321,
-      "ثُ": 322,
-      "خِ": 323,
-      "الخَ": 324,
-      "بٍ": 325,
-      "باً": 326,
-      "خا": 327,
-      "غِ": 328,
-      "الَّ": 329,
-      "الفَ": 330,
-      "وَإِن": 331,
-      "رٌ": 332,
-      "طُ": 333,
-      "وَقَد": 334,
-      "فيهِ": 335,
-      "حَتّى": 336,
-      "وَلَ": 337,
-      "الرِ": 338,
-      "عِن": 339,
-      "ينِ": 340,
-      "الُ": 341,
-      "أَب": 342,
-      "نو": 343,
-      "كُم": 344,
-      "طو": 345,
-      "لٍ": 346,
-      "اللَي": 347,
-      "أَم": 348,
-      "ةُ": 349,
-      "لاً": 350,
-      "الهَ": 351,
-      "جي": 352,
-      "نى": 353,
-      "الزَما": 354,
-      "كَم": 355,
-      "مٌ": 356,
-      "مِنهُ": 357,
-      "الطَ": 358,
-      "بٌ": 359,
-      "الصَ": 360,
-      "النا": 361,
-      "مى": 362,
-      "لَها": 363,
-      "طي": 364,
-      "عو": 365,
-      "يّا": 366,
-      "قى": 367,
-      "إِلَي": 368,
-      "يَن": 369,
-      "وُ": 370,
-      "الدَه": 371,
-      "وَمَ": 372,
-      "وَّ": 373,
-      "يَومَ": 374,
-      "الَّذي": 375,
-      "المَج": 376,
-      "رُّ": 377,
-      "كُلَّ": 378,
-      "يِّ": 379,
-      "فو": 380,
-      "عاً": 381,
-      "ناً": 382,
-      "بو": 383,
-      "سو": 384,
-      "الا": 385,
-      "أَح": 386,
-      "تُهُ": 387,
-      "وارِ": 388,
-      "حي": 389,
-      "الدُ": 390,
-      "meter_0": 391,
-      "لٌ": 392,
-      "قَو": 393,
-      "قَب": 394,
-      "وَج": 395,
-      "دُّ": 396,
-      "مَع": 397,
-      "دٌ": 398,
-      "أَنتَ": 399,
-      "الحُ": 400,
-      "العُلى": 401,
-      "لُّ": 402,
-      "meter_3": 403,
-      "التَ": 404,
-      "عَي": 405,
-      "عَنِ": 406,
-      "تى": 407,
-      "ثا": 408,
-      "وَلَم": 409,
-      "غَيرِ": 410,
-      "وَمُ": 411,
-      "قٍ": 412,
-      "عِندَ": 413,
-      "بى": 414,
-      "رَت": 415,
-      "ضى": 416,
-      "تَح": 417,
-      "وائِ": 418,
-      "زُ": 419,
-      "أَق": 420,
-      "بَي": 421,
-      "اِن": 422,
-      "هُمُ": 423,
-      "أَس": 424,
-      "فَما": 425,
-      "ثِ": 426,
-      "كَما": 427,
-      "كُلُّ": 428,
-      "لِّ": 429,
-      "وَنَ": 430,
-      "أَط": 431,
-      "وَرَ": 432,
-      "مَو": 433,
-      "هِم": 434,
-      "دِّ": 435,
-      "رِّ": 436,
-      "نٍ": 437,
-      "الإِ": 438,
-      "تَهُ": 439,
-      "لَيسَ": 440,
-      "وَلَو": 441,
-      "غَيرَ": 442,
-      "ٍّ": 443,
-      "مِث": 444,
-      "وَه": 445,
-      "نَف": 446,
-      "الأَر": 447,
-      "القُ": 448,
-      "مُن": 449,
-      "دونَ": 450,
-      "الحِ": 451,
-      "الأَع": 452,
-      "كَي": 453,
-      "عَلي": 454,
-      "عٌ": 455,
-      "عٍ": 456,
-      "اللَ": 457,
-      "ماءِ": 458,
-      "أَل": 459,
-      "تِهِ": 460,
-      "النُ": 461,
-      "الكَ": 462,
-      "عَز": 463,
-      "كُنتُ": 464,
-      "شي": 465,
-      "أَنا": 466,
-      "حَي": 467,
-      "ذاكَ": 468,
-      "يَع": 469,
-      "هَل": 470,
-      "صو": 471,
-      "الضَ": 472,
-      "لَكَ": 473,
-      "أَخ": 474,
-      "مُتَ": 475,
-      "وَق": 476,
-      "مَر": 477,
-      "يَر": 478,
-      "وَجَ": 479,
-      "ٌّ": 480,
-      "الأُ": 481,
-      "فيها": 482,
-      "رَأ": 483,
-      "قُل": 484,
-      "فَلا": 485,
-      "فَاِ": 486,
-      "القَنا": 487,
-      "مَج": 488,
-      "هَذا": 489,
-      "قاً": 490,
-      "قَلبي": 491,
-      "السُ": 492,
-      "عَت": 493,
-      "سَي": 494,
-      "فَّ": 495,
-      "الجِ": 496,
-      "الخُ": 497,
-      "غَيرُ": 498,
-      "ظا": 499,
-      "هو": 500,
-      "الرُ": 501,
-      "دَم": 502,
-      "كُمُ": 503,
-      "كو": 504,
-      "وَعَ": 505,
-      "المَع": 506,
-      "اللَيالي": 507,
-      "نٌ": 508,
-      "غي": 509,
-      "أَج": 510,
-      "مِّ": 511,
-      "مُّ": 512,
-      "وَقَ": 513,
-      "الظَ": 514,
-      "تَر": 515,
-      "تَن": 516,
-      "دَت": 517,
-      "لتَ": 518,
-      "الآ": 519,
-      "طَو": 520,
-      "البُ": 521,
-      "يَوماً": 522,
-      "وَمِن": 523,
-      "وَيُ": 524,
-      "زي": 525,
-      "مَط": 526,
-      "تَرَ": 527,
-      "خَي": 528
+      "<|meter_14|>": 19,
+      "<|meter_15|>": 20,
+      "<|meter_16|>": 21,
+      "<|meter_17|>": 22,
+      "<|meter_18|>": 23,
+      "<|meter_19|>": 24,
+      " ": 25,
+      "0": 26,
+      "1": 27,
+      "2": 28,
+      "3": 29,
+      "4": 30,
+      "5": 31,
+      "6": 32,
+      "7": 33,
+      "8": 34,
+      "9": 35,
+      ":": 36,
+      "<": 37,
+      ">": 38,
+      "_": 39,
+      "b": 40,
+      "e": 41,
+      "m": 42,
+      "p": 43,
+      "r": 44,
+      "s": 45,
+      "t": 46,
+      "v": 47,
+      "|": 48,
+      "ء": 49,
+      "آ": 50,
+      "أ": 51,
+      "ؤ": 52,
+      "إ": 53,
+      "ئ": 54,
+      "ا": 55,
+      "ب": 56,
+      "ة": 57,
+      "ت": 58,
+      "ث": 59,
+      "ج": 60,
+      "ح": 61,
+      "خ": 62,
+      "د": 63,
+      "ذ": 64,
+      "ر": 65,
+      "ز": 66,
+      "س": 67,
+      "ش": 68,
+      "ص": 69,
+      "ض": 70,
+      "ط": 71,
+      "ظ": 72,
+      "ع": 73,
+      "غ": 74,
+      "ف": 75,
+      "ق": 76,
+      "ك": 77,
+      "ل": 78,
+      "م": 79,
+      "ن": 80,
+      "ه": 81,
+      "و": 82,
+      "ى": 83,
+      "ي": 84,
+      "ً": 85,
+      "ٌ": 86,
+      "ٍ": 87,
+      "َ": 88,
+      "ُ": 89,
+      "ِ": 90,
+      "ّ": 91,
+      "ْ": 92
     },
-    "merges": [
-      "< |",
-      "| >",
-      "e p",
-      "s ep",
-      "ا ل",
-      "و َ",
-      "b sep",
-      "v sep",
-      "أ َ",
-      "م َ",
-      "ع َ",
-      "ل َ",
-      "ت َ",
-      "م ِ",
-      "ن َ",
-      "ّ َ",
-      "ب ِ",
-      "م ا",
-      "ي َ",
-      "ر َ",
-      "م ُ",
-      "ق َ",
-      "ه ُ",
-      "ك َ",
-      "ف َ",
-      "ب َ",
-      "إ ِ",
-      "ه ا",
-      "ر ِ",
-      "ل ِ",
-      "ل ا",
-      "ن ا",
-      "و ا",
-      "د َ",
-      "ه ِ",
-      "مِ ن",
-      "ف ي",
-      "ح َ",
-      "ر ا",
-      "ن ِ",
-      "ل ى",
-      "س َ",
-      "ت ُ",
-      "ج َ",
-      "ر ُ",
-      "د ِ",
-      "ع ِ",
-      "ع ُ",
-      "ي ا",
-      "غ َ",
-      "ك ُ",
-      "أَ ن",
-      "د ا",
-      "ل ُ",
-      "ّ ِ",
-      "ب ا",
-      "ال أَ",
-      "ذ ا",
-      "ط َ",
-      "ّ ا",
-      "p sep",
-      "ّ ُ",
-      "ن ي",
-      "ال مَ",
-      "ع ا",
-      "ه َ",
-      "د ُ",
-      "ش َ",
-      "ب ُ",
-      "ي ُ",
-      "عَ لى",
-      "ق ِ",
-      "وَ ال",
-      "ن ُ",
-      "ق ا",
-      "لَ ي",
-      "إِ ن",
-      "د ي",
-      "ق ُ",
-      "خ َ",
-      "ئ ِ",
-      "ا ِ",
-      "ض َ",
-      "ص َ",
-      "ل ي",
-      "بِ ال",
-      "ر ي",
-      "ز َ",
-      "ت ِ",
-      "ب ي",
-      "إِ ذا",
-      "س ِ",
-      "ح ِ",
-      "ف ِ",
-      "س ا",
-      "قَ د",
-      "ك ا",
-      "عَ ن",
-      "لَ م",
-      "ج ِ",
-      "وَ أَ",
-      "ح ُ",
-      "ج ا",
-      "ل َّ",
-      "ف ا",
-      "ال ِ",
-      "م ي",
-      "أ ُ",
-      "مَ ن",
-      "e r",
-      "e t",
-      "m et",
-      "er _",
-      "met er_",
-      "مِ نَ",
-      "ي نَ",
-      "ف ُ",
-      "ال مُ",
-      "ال عَ",
-      "ة ٍ",
-      "وَ لا",
-      "ط ا",
-      "ح ا",
-      "س ُ",
-      "لِ ل",
-      "أَن َّ",
-      "ء ِ",
-      "ذ َ",
-      "غَ ي",
-      "ال قَ",
-      "ة ً",
-      "إِ لى",
-      "أَ و",
-      "ال نَ",
-      "ك ِ",
-      "ط ِ",
-      "ر ى",
-      "ض ِ",
-      "بَ ع",
-      "ء َ",
-      "ل و",
-      "يَ و",
-      "كُ ل",
-      "ث َ",
-      "ر و",
-      "د و",
-      "ر َّ",
-      "بِ هِ",
-      "س تَ",
-      "لَ هُ",
-      "ق ي",
-      "ص ا",
-      "لَ و",
-      "ال ي",
-      "ظ َ",
-      "ل ّا",
-      "ج ُ",
-      "ال رَ",
-      "ّ ي",
-      "ع ي",
-      "ال َ",
-      "ز ِ",
-      "ض ا",
-      "ة ٌ",
-      "ن َّ",
-      "غ ُ",
-      "ّ ى",
-      "ش ا",
-      "غ ا",
-      "و ى",
-      "ة ِ",
-      "د ى",
-      "هُ م",
-      "أَ ع",
-      "م َّ",
-      "ص ِ",
-      "ذ ي",
-      "إِن َّ",
-      "ال عُ",
-      "ما ً",
-      "ال حَ",
-      "كَ أَنَّ",
-      "ز ا",
-      "إِ لّا",
-      "ال سَ",
-      "وَ ما",
-      "ش ِ",
-      "كا نَ",
-      "أَ ي",
-      "ال جَ",
-      "قَ ل",
-      "م ٍ",
-      "ء ُ",
-      "خ ُ",
-      "حَ ت",
-      "بَ ينَ",
-      "ال دَ",
-      "ر ٍ",
-      "د ٍ",
-      "ض ُ",
-      "ج و",
-      "ذ ُ",
-      "ي َّ",
-      "م و",
-      "دا ً",
-      "ال شَ",
-      "زَ ما",
-      "ة َ",
-      "را ً",
-      "ت ي",
-      "ذ ِ",
-      "بَع دَ",
-      "د َّ",
-      "ق و",
-      "وَ يَ",
-      "وَ تَ",
-      "ظ ُ",
-      "ش ُ",
-      "ي و",
-      "بِ ها",
-      "ال بَ",
-      "ظ ِ",
-      "ال غَ",
-      "ض ي",
-      "ال عِ",
-      "كُ ن",
-      "ال وَ",
-      "س ي",
-      "و ِ",
-      "ب َّ",
-      "وَ اِ",
-      "ص ُ",
-      "عَ لَي",
-      "كُل ِّ",
-      "أَ ر",
-      "ت ا",
-      "ث ُ",
-      "خ ِ",
-      "ال خَ",
-      "ب ٍ",
-      "با ً",
-      "خ ا",
-      "غ ِ",
-      "ال َّ",
-      "ال فَ",
-      "وَ إِن",
-      "ر ٌ",
-      "ط ُ",
-      "وَ قَد",
-      "في هِ",
-      "حَت ّى",
-      "وَ لَ",
-      "ال رِ",
-      "عِ ن",
-      "ي نِ",
-      "ال ُ",
-      "أَ ب",
-      "ن و",
-      "كُ م",
-      "ط و",
-      "ل ٍ",
-      "ال لَي",
-      "أَ م",
-      "ة ُ",
-      "لا ً",
-      "ال هَ",
-      "ج ي",
-      "ن ى",
-      "ال زَما",
-      "كَ م",
-      "م ٌ",
-      "مِن هُ",
-      "ال طَ",
-      "ب ٌ",
-      "ال صَ",
-      "ال نا",
-      "م ى",
-      "لَ ها",
-      "ط ي",
-      "ع و",
-      "ي ّا",
-      "ق ى",
-      "إِ لَي",
-      "يَ ن",
-      "و ُ",
-      "الدَ ه",
-      "وَ مَ",
-      "و َّ",
-      "يَو مَ",
-      "الَّ ذي",
-      "المَ ج",
-      "ر ُّ",
-      "كُ لَّ",
-      "ي ِّ",
-      "ف و",
-      "عا ً",
-      "نا ً",
-      "ب و",
-      "س و",
-      "ال ا",
-      "أَ ح",
-      "تُ هُ",
-      "وا رِ",
-      "ح ي",
-      "ال دُ",
-      "meter_ 0",
-      "ل ٌ",
-      "قَ و",
-      "قَ ب",
-      "وَ ج",
-      "د ُّ",
-      "مَ ع",
-      "د ٌ",
-      "أَن تَ",
-      "ال حُ",
-      "العُ لى",
-      "ل ُّ",
-      "meter_ 3",
-      "ال تَ",
-      "عَ ي",
-      "عَ نِ",
-      "ت ى",
-      "ث ا",
-      "وَ لَم",
-      "غَي رِ",
-      "وَ مُ",
-      "ق ٍ",
-      "عِن دَ",
-      "ب ى",
-      "رَ ت",
-      "ض ى",
-      "تَ ح",
-      "وا ئِ",
-      "ز ُ",
-      "أَ ق",
-      "بَ ي",
-      "اِ ن",
-      "هُ مُ",
-      "أَ س",
-      "فَ ما",
-      "ث ِ",
-      "كَ ما",
-      "كُل ُّ",
-      "ل ِّ",
-      "وَ نَ",
-      "أَ ط",
-      "وَ رَ",
-      "مَ و",
-      "هِ م",
-      "د ِّ",
-      "ر ِّ",
-      "ن ٍ",
-      "ال إِ",
-      "تَ هُ",
-      "لَي سَ",
-      "وَ لَو",
-      "غَي رَ",
-      "ّ ٍ",
-      "مِ ث",
-      "وَ ه",
-      "نَ ف",
-      "الأَ ر",
-      "ال قُ",
-      "مُ ن",
-      "دو نَ",
-      "ال حِ",
-      "الأَ ع",
-      "كَ ي",
-      "عَ لي",
-      "ع ٌ",
-      "ع ٍ",
-      "ال لَ",
-      "ما ءِ",
-      "أَ ل",
-      "تِ هِ",
-      "ال نُ",
-      "ال كَ",
-      "عَ ز",
-      "كُن تُ",
-      "ش ي",
-      "أَ نا",
-      "حَ ي",
-      "ذا كَ",
-      "يَ ع",
-      "هَ ل",
-      "ص و",
-      "ال ضَ",
-      "لَ كَ",
-      "أَ خ",
-      "مُ تَ",
-      "وَ ق",
-      "مَ ر",
-      "يَ ر",
-      "وَ جَ",
-      "ّ ٌ",
-      "ال أُ",
-      "في ها",
-      "رَ أ",
-      "قُ ل",
-      "فَ لا",
-      "فَ اِ",
-      "القَ نا",
-      "مَ ج",
-      "هَ ذا",
-      "قا ً",
-      "قَل بي",
-      "ال سُ",
-      "عَ ت",
-      "سَ ي",
-      "ف َّ",
-      "ال جِ",
-      "ال خُ",
-      "غَي رُ",
-      "ظ ا",
-      "ه و",
-      "ال رُ",
-      "دَ م",
-      "كُ مُ",
-      "ك و",
-      "وَ عَ",
-      "المَ ع",
-      "اللَي الي",
-      "ن ٌ",
-      "غ ي",
-      "أَ ج",
-      "م ِّ",
-      "م ُّ",
-      "وَ قَ",
-      "ال ظَ",
-      "تَ ر",
-      "تَ ن",
-      "دَ ت",
-      "ل تَ",
-      "ال آ",
-      "طَ و",
-      "ال بُ",
-      "يَو ماً",
-      "وَ مِن",
-      "وَ يُ",
-      "ز ي",
-      "مَ ط",
-      "تَ رَ",
-      "خَ ي"
-    ]
+    "merges": []
   }
 }

vocab.json

@@ -1 +1 @@
-{"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"|psep|":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18,"0":19,"1":20,"2":21,"3":22,"4":23,"5":24,"6":25,"7":26,"8":27,"9":28,":":29,"<":30,">":31,"_":32,"b":33,"e":34,"m":35,"p":36,"r":37,"s":38,"t":39,"v":40,"|":41,"ء":42,"آ":43,"أ":44,"ؤ":45,"إ":46,"ئ":47,"ا":48,"ب":49,"ة":50,"ت":51,"ث":52,"ج":53,"ح":54,"خ":55,"د":56,"ذ":57,"ر":58,"ز":59,"س":60,"ش":61,"ص":62,"ض":63,"ط":64,"ظ":65,"ع":66,"غ":67,"ف":68,"ق":69,"ك":70,"ل":71,"م":72,"ن":73,"ه":74,"و":75,"ى":76,"ي":77,"ً":78,"ٌ":79,"ٍ":80,"َ":81,"ُ":82,"ِ":83,"ّ":84,"ْ":85,"<|":86,"|>":87,"ep":88,"sep":89,"ال":90,"وَ":91,"bsep":92,"vsep":93,"أَ":94,"مَ":95,"عَ":96,"لَ":97,"تَ":98,"مِ":99,"نَ":100,"َّ":101,"بِ":102,"ما":103,"يَ":104,"رَ":105,"مُ":106,"قَ":107,"هُ":108,"كَ":109,"فَ":110,"بَ":111,"إِ":112,"ها":113,"رِ":114,"لِ":115,"لا":116,"نا":117,"وا":118,"دَ":119,"هِ":120,"مِن":121,"في":122,"حَ":123,"را":124,"نِ":125,"لى":126,"سَ":127,"تُ":128,"جَ":129,"رُ":130,"دِ":131,"عِ":132,"عُ":133,"يا":134,"غَ":135,"كُ":136,"أَن":137,"دا":138,"لُ":139,"ِّ":140,"با":141,"الأَ":142,"ذا":143,"طَ":144,"ّا":145,"psep":146,"ُّ":147,"ني":148,"المَ":149,"عا":150,"هَ":151,"دُ":152,"شَ":153,"بُ":154,"يُ":155,"عَلى":156,"قِ":157,"وَال":158,"نُ":159,"قا":160,"لَي":161,"إِن":162,"دي":163,"قُ":164,"خَ":165,"ئِ":166,"اِ":167,"ضَ":168,"صَ":169,"لي":170,"بِال":171,"ري":172,"زَ":173,"تِ":174,"بي":175,"إِذا":176,"سِ":177,"حِ":178,"فِ":179,"سا":180,"قَد":181,"كا":182,"عَن":183,"لَم":184,"جِ":185,"وَأَ":186,"حُ":187,"جا":188,"لَّ":189,"فا":190,"الِ":191,"مي":192,"أُ":193,"مَن":194,"er":195,"et":196,"met":197,"er_":198,"meter_":199,"مِنَ":200,"ينَ":201,"فُ":202,"المُ":203,"العَ":204,"ةٍ":205,"وَلا":206,"طا":207,"حا":208,"سُ":209,"لِل":210,"أَنَّ":211,"ءِ":212,"ذَ":213,"غَي":214,"القَ":215,"ةً":216,"إِلى":217,"أَو":218,"النَ":219,"كِ":220,"طِ":221,"رى":222,"ضِ":223,"بَع":224,"ءَ":225,"لو":226,"يَو":227,"كُل":228,"ثَ":229,"رو":230,"دو":231,"رَّ":232,"بِهِ":233,"ستَ":234,"لَهُ":235,"قي":236,"صا":237,"لَو":238,"الي":239,"ظَ":240,"لّا":241,"جُ":242,"الرَ":243,"ّي":244,"عي":245,"الَ":246,"زِ":247,"ضا":248,"ةٌ":249,"نَّ":250,"غُ":251,"ّى":252,"شا":253,"غا":254,"وى":255,"ةِ":256,"دى":257,"هُم":258,"أَع":259,"مَّ":260,"صِ":261,"ذي":262,"إِنَّ":263,"العُ":264,"ماً":265,"الحَ":266,"كَأَنَّ":267,"زا":268,"إِلّا":269,"السَ":270,"وَما":271,"شِ":272,"كانَ":273,"أَي":274,"الجَ":275,"قَل":276,"مٍ":277,"ءُ":278,"خُ":279,"حَت":280,"بَينَ":281,"الدَ":282,"رٍ":283,"دٍ":284,"ضُ":285,"جو":286,"ذُ":287,"يَّ":288,"مو":289,"داً":290,"الشَ":291,"زَما":292,"ةَ":293,"راً":294,"تي":295,"ذِ":296,"بَعدَ":297,"دَّ":298,"قو":299,"وَيَ":300,"وَتَ":301,"ظُ":302,"شُ":303,"يو":304,"بِها":305,"البَ":306,"ظِ":307,"الغَ":308,"ضي":309,"العِ":310,"كُن":311,"الوَ":312,"سي":313,"وِ":314,"بَّ":315,"وَاِ":316,"صُ":317,"عَلَي":318,"كُلِّ":319,"أَر":320,"تا":321,"ثُ":322,"خِ":323,"الخَ":324,"بٍ":325,"باً":326,"خا":327,"غِ":328,"الَّ":329,"الفَ":330,"وَإِن":331,"رٌ":332,"طُ":333,"وَقَد":334,"فيهِ":335,"حَتّى":336,"وَلَ":337,"الرِ":338,"عِن":339,"ينِ":340,"الُ":341,"أَب":342,"نو":343,"كُم":344,"طو":345,"لٍ":346,"اللَي":347,"أَم":348,"ةُ":349,"لاً":350,"الهَ":351,"جي":352,"نى":353,"الزَما":354,"كَم":355,"مٌ":356,"مِنهُ":357,"الطَ":358,"بٌ":359,"الصَ":360,"النا":361,"مى":362,"لَها":363,"طي":364,"عو":365,"يّا":366,"قى":367,"إِلَي":368,"يَن":369,"وُ":370,"الدَه":371,"وَمَ":372,"وَّ":373,"يَومَ":374,"الَّذي":375,"المَج":376,"رُّ":377,"كُلَّ":378,"يِّ":379,"فو":380,"عاً":381,"ناً":382,"بو":383,"سو":384,"الا":385,"أَح":386,"تُهُ":387,"وارِ":388,"حي":389,"الدُ":390,"meter_0":391,"لٌ":392,"قَو":393,"قَب":394,"وَج":395,"دُّ":396,"مَع":397,"دٌ":398,"أَنتَ":399,"الحُ":400,"العُلى":401,"لُّ":402,"meter_3":403,"التَ":404,"عَي":405,"عَنِ":406,"تى":407,"ثا":408,"وَلَم":409,"غَيرِ":410,"وَمُ":411,"قٍ":412,"عِندَ":413,"بى":414,"رَت":415,"ضى":416,"تَح":417,"وائِ":418,"زُ":419,"أَق":420,"بَي":421,"اِن":422,"هُمُ":423,"أَس":424,"فَما":425,"ثِ":426,"كَما":427,"كُلُّ":428,"لِّ":429,"وَنَ":430,"أَط":431,"وَرَ":432,"مَو":433,"هِم":434,"دِّ":435,"رِّ":436,"نٍ":437,"الإِ":438,"تَهُ":439,"لَيسَ":440,"وَلَو":441,"غَيرَ":442,"ٍّ":443,"مِث":444,"وَه":445,"نَف":446,"الأَر":447,"القُ":448,"مُن":449,"دونَ":450,"الحِ":451,"الأَع":452,"كَي":453,"عَلي":454,"عٌ":455,"عٍ":456,"اللَ":457,"ماءِ":458,"أَل":459,"تِهِ":460,"النُ":461,"الكَ":462,"عَز":463,"كُنتُ":464,"شي":465,"أَنا":466,"حَي":467,"ذاكَ":468,"يَع":469,"هَل":470,"صو":471,"الضَ":472,"لَكَ":473,"أَخ":474,"مُتَ":475,"وَق":476,"مَر":477,"يَر":478,"وَجَ":479,"ٌّ":480,"الأُ":481,"فيها":482,"رَأ":483,"قُل":484,"فَلا":485,"فَاِ":486,"القَنا":487,"مَج":488,"هَذا":489,"قاً":490,"قَلبي":491,"السُ":492,"عَت":493,"سَي":494,"فَّ":495,"الجِ":496,"الخُ":497,"غَيرُ":498,"ظا":499,"هو":500,"الرُ":501,"دَم":502,"كُمُ":503,"كو":504,"وَعَ":505,"المَع":506,"اللَيالي":507,"نٌ":508,"غي":509,"أَج":510,"مِّ":511,"مُّ":512,"وَقَ":513,"الظَ":514,"تَر":515,"تَن":516,"دَت":517,"لتَ":518,"الآ":519,"طَو":520,"البُ":521,"يَوماً":522,"وَمِن":523,"وَيُ":524,"زي":525,"مَط":526,"تَرَ":527,"خَي":528}
+{"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"|psep|":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18,"<|meter_14|>":19,"<|meter_15|>":20,"<|meter_16|>":21,"<|meter_17|>":22,"<|meter_18|>":23,"<|meter_19|>":24," ":25,"0":26,"1":27,"2":28,"3":29,"4":30,"5":31,"6":32,"7":33,"8":34,"9":35,":":36,"<":37,">":38,"_":39,"b":40,"e":41,"m":42,"p":43,"r":44,"s":45,"t":46,"v":47,"|":48,"ء":49,"آ":50,"أ":51,"ؤ":52,"إ":53,"ئ":54,"ا":55,"ب":56,"ة":57,"ت":58,"ث":59,"ج":60,"ح":61,"خ":62,"د":63,"ذ":64,"ر":65,"ز":66,"س":67,"ش":68,"ص":69,"ض":70,"ط":71,"ظ":72,"ع":73,"غ":74,"ف":75,"ق":76,"ك":77,"ل":78,"م":79,"ن":80,"ه":81,"و":82,"ى":83,"ي":84,"ً":85,"ٌ":86,"ٍ":87,"َ":88,"ُ":89,"ِ":90,"ّ":91,"ْ":92}