{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "<|vsep|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "<|bsep|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "<|pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "|psep|", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 5, "content": "<|meter_0|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 6, "content": "<|meter_1|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 7, "content": "<|meter_2|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 8, "content": "<|meter_3|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 9, "content": "<|meter_4|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 10, "content": "<|meter_5|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 11, "content": "<|meter_6|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 12, "content": "<|meter_7|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 13, "content": "<|meter_8|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 14, "content": "<|meter_9|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 15, "content": "<|meter_10|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 16, "content": "<|meter_11|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 17, "content": "<|meter_12|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 18, "content": "<|meter_13|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 19, "content": "<|meter_14|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 20, "content": "<|meter_15|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 21, "content": "<|meter_16|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 22, "content": "<|meter_17|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 23, "content": "<|meter_18|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 24, "content": "<|meter_19|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "UnicodeScripts" }, "post_processor": null, "decoder": null, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "vocab": { "<|endoftext|>": 0, "<|vsep|>": 1, "<|bsep|>": 2, "<|pad|>": 3, "|psep|": 4, "<|meter_0|>": 5, "<|meter_1|>": 6, "<|meter_2|>": 7, "<|meter_3|>": 8, "<|meter_4|>": 9, "<|meter_5|>": 10, "<|meter_6|>": 11, "<|meter_7|>": 12, "<|meter_8|>": 13, "<|meter_9|>": 14, "<|meter_10|>": 15, "<|meter_11|>": 16, "<|meter_12|>": 17, "<|meter_13|>": 18, "<|meter_14|>": 19, "<|meter_15|>": 20, "<|meter_16|>": 21, "<|meter_17|>": 22, "<|meter_18|>": 23, "<|meter_19|>": 24, " ": 25, "0": 26, "1": 27, "2": 28, "3": 29, "4": 30, "5": 31, "6": 32, "7": 33, "8": 34, "9": 35, ":": 36, "<": 37, ">": 38, "_": 39, "b": 40, "e": 41, "m": 42, "p": 43, "r": 44, "s": 45, "t": 46, "v": 47, "|": 48, "ء": 49, "آ": 50, "أ": 51, "ؤ": 52, "إ": 53, "ئ": 54, "ا": 55, "ب": 56, "ة": 57, "ت": 58, "ث": 59, "ج": 60, "ح": 61, "خ": 62, "د": 63, "ذ": 64, "ر": 65, "ز": 66, "س": 67, "ش": 68, "ص": 69, "ض": 70, "ط": 71, "ظ": 72, "ع": 73, "غ": 74, "ف": 75, "ق": 76, "ك": 77, "ل": 78, "م": 79, "ن": 80, "ه": 81, "و": 82, "ى": 83, "ي": 84, "ً": 85, "ٌ": 86, "ٍ": 87, "َ": 88, "ُ": 89, "ِ": 90, "ّ": 91, "ْ": 92 }, "merges": [] } }