tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 0,
      "content": "<|endoftext|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 1,
      "content": "<|vsep|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 2,
      "content": "<|bsep|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 3,
      "content": "<|pad|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 4,
      "content": "|psep|",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 5,
      "content": "<|meter_0|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 6,
      "content": "<|meter_1|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 7,
      "content": "<|meter_2|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 8,
      "content": "<|meter_3|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 9,
      "content": "<|meter_4|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 10,
      "content": "<|meter_5|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 11,
      "content": "<|meter_6|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 12,
      "content": "<|meter_7|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 13,
      "content": "<|meter_8|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 14,
      "content": "<|meter_9|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 15,
      "content": "<|meter_10|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 16,
      "content": "<|meter_11|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 17,
      "content": "<|meter_12|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 18,
      "content": "<|meter_13|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 19,
      "content": "<|meter_14|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 20,
      "content": "<|meter_15|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 21,
      "content": "<|meter_16|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 22,
      "content": "<|meter_17|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 23,
      "content": "<|meter_18|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 24,
      "content": "<|meter_19|>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "UnicodeScripts"
  },
  "post_processor": null,
  "decoder": null,
  "model": {
    "type": "BPE",
    "dropout": null,
    "unk_token": null,
    "continuing_subword_prefix": null,
    "end_of_word_suffix": null,
    "fuse_unk": false,
    "byte_fallback": false,
    "vocab": {
      "<|endoftext|>": 0,
      "<|vsep|>": 1,
      "<|bsep|>": 2,
      "<|pad|>": 3,
      "|psep|": 4,
      "<|meter_0|>": 5,
      "<|meter_1|>": 6,
      "<|meter_2|>": 7,
      "<|meter_3|>": 8,
      "<|meter_4|>": 9,
      "<|meter_5|>": 10,
      "<|meter_6|>": 11,
      "<|meter_7|>": 12,
      "<|meter_8|>": 13,
      "<|meter_9|>": 14,
      "<|meter_10|>": 15,
      "<|meter_11|>": 16,
      "<|meter_12|>": 17,
      "<|meter_13|>": 18,
      "<|meter_14|>": 19,
      "<|meter_15|>": 20,
      "<|meter_16|>": 21,
      "<|meter_17|>": 22,
      "<|meter_18|>": 23,
      "<|meter_19|>": 24,
      " ": 25,
      "0": 26,
      "1": 27,
      "2": 28,
      "3": 29,
      "4": 30,
      "5": 31,
      "6": 32,
      "7": 33,
      "8": 34,
      "9": 35,
      ":": 36,
      "<": 37,
      ">": 38,
      "_": 39,
      "b": 40,
      "e": 41,
      "m": 42,
      "p": 43,
      "r": 44,
      "s": 45,
      "t": 46,
      "v": 47,
      "|": 48,
      "ء": 49,
      "آ": 50,
      "أ": 51,
      "ؤ": 52,
      "إ": 53,
      "ئ": 54,
      "ا": 55,
      "ب": 56,
      "ة": 57,
      "ت": 58,
      "ث": 59,
      "ج": 60,
      "ح": 61,
      "خ": 62,
      "د": 63,
      "ذ": 64,
      "ر": 65,
      "ز": 66,
      "س": 67,
      "ش": 68,
      "ص": 69,
      "ض": 70,
      "ط": 71,
      "ظ": 72,
      "ع": 73,
      "غ": 74,
      "ف": 75,
      "ق": 76,
      "ك": 77,
      "ل": 78,
      "م": 79,
      "ن": 80,
      "ه": 81,
      "و": 82,
      "ى": 83,
      "ي": 84,
      "ً": 85,
      "ٌ": 86,
      "ٍ": 87,
      "َ": 88,
      "ُ": 89,
      "ِ": 90,
      "ّ": 91,
      "ْ": 92
    },
    "merges": []
  }
}