| { | |
| "version": "1.0", | |
| "truncation": null, | |
| "padding": null, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "<|endoftext|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 1, | |
| "content": "<|psep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 2, | |
| "content": "</|psep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 3, | |
| "content": "<|vsep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 4, | |
| "content": "<|bsep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 5, | |
| "content": "</|bsep|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 6, | |
| "content": "<|pad|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 7, | |
| "content": "<|meter_0|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 8, | |
| "content": "<|meter_1|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 9, | |
| "content": "<|meter_2|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 10, | |
| "content": "<|meter_3|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 11, | |
| "content": "<|meter_4|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 12, | |
| "content": "<|meter_5|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 13, | |
| "content": "<|meter_6|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 14, | |
| "content": "<|meter_7|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 15, | |
| "content": "<|meter_8|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 16, | |
| "content": "<|meter_9|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 17, | |
| "content": "<|meter_10|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 18, | |
| "content": "<|meter_11|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 19, | |
| "content": "<|meter_12|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 20, | |
| "content": "<|meter_13|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 21, | |
| "content": "<|meter_14|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 22, | |
| "content": "<|meter_15|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 23, | |
| "content": "<|res_0|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 24, | |
| "content": "<|res_1|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 25, | |
| "content": "<|res_2|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 26, | |
| "content": "<|res_3|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 27, | |
| "content": "<|res_4|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 28, | |
| "content": "<|res_5|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 29, | |
| "content": "<|res_6|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 30, | |
| "content": "<|res_7|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 31, | |
| "content": "<|res_8|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 32, | |
| "content": "<|res_9|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| } | |
| ], | |
| "normalizer": null, | |
| "pre_tokenizer": { | |
| "type": "UnicodeScripts" | |
| }, | |
| "post_processor": null, | |
| "decoder": null, | |
| "model": { | |
| "type": "BPE", | |
| "dropout": null, | |
| "unk_token": null, | |
| "continuing_subword_prefix": null, | |
| "end_of_word_suffix": null, | |
| "fuse_unk": false, | |
| "byte_fallback": false, | |
| "vocab": { | |
| "<|endoftext|>": 0, | |
| "<|psep|>": 1, | |
| "</|psep|>": 2, | |
| "<|vsep|>": 3, | |
| "<|bsep|>": 4, | |
| "</|bsep|>": 5, | |
| "<|pad|>": 6, | |
| "<|meter_0|>": 7, | |
| "<|meter_1|>": 8, | |
| "<|meter_2|>": 9, | |
| "<|meter_3|>": 10, | |
| "<|meter_4|>": 11, | |
| "<|meter_5|>": 12, | |
| "<|meter_6|>": 13, | |
| "<|meter_7|>": 14, | |
| "<|meter_8|>": 15, | |
| "<|meter_9|>": 16, | |
| "<|meter_10|>": 17, | |
| "<|meter_11|>": 18, | |
| "<|meter_12|>": 19, | |
| "<|meter_13|>": 20, | |
| "<|meter_14|>": 21, | |
| "<|meter_15|>": 22, | |
| "<|res_0|>": 23, | |
| "<|res_1|>": 24, | |
| "<|res_2|>": 25, | |
| "<|res_3|>": 26, | |
| "<|res_4|>": 27, | |
| "<|res_5|>": 28, | |
| "<|res_6|>": 29, | |
| "<|res_7|>": 30, | |
| "<|res_8|>": 31, | |
| "<|res_9|>": 32, | |
| " ": 33, | |
| "/": 34, | |
| "0": 35, | |
| "1": 36, | |
| "2": 37, | |
| "3": 38, | |
| "4": 39, | |
| "5": 40, | |
| "6": 41, | |
| "7": 42, | |
| "8": 43, | |
| "9": 44, | |
| "<": 45, | |
| ">": 46, | |
| "_": 47, | |
| "b": 48, | |
| "e": 49, | |
| "m": 50, | |
| "p": 51, | |
| "r": 52, | |
| "s": 53, | |
| "t": 54, | |
| "v": 55, | |
| "|": 56, | |
| "~": 57, | |
| "ء": 58, | |
| "أ": 59, | |
| "ؤ": 60, | |
| "ئ": 61, | |
| "ا": 62, | |
| "ب": 63, | |
| "ة": 64, | |
| "ت": 65, | |
| "ث": 66, | |
| "ج": 67, | |
| "ح": 68, | |
| "خ": 69, | |
| "د": 70, | |
| "ذ": 71, | |
| "ر": 72, | |
| "ز": 73, | |
| "س": 74, | |
| "ش": 75, | |
| "ص": 76, | |
| "ض": 77, | |
| "ط": 78, | |
| "ظ": 79, | |
| "ع": 80, | |
| "غ": 81, | |
| "ف": 82, | |
| "ق": 83, | |
| "ك": 84, | |
| "ل": 85, | |
| "م": 86, | |
| "ن": 87, | |
| "ه": 88, | |
| "و": 89, | |
| "ى": 90, | |
| "ي": 91, | |
| "ً": 92, | |
| "ٌ": 93, | |
| "ٍ": 94, | |
| "َ": 95, | |
| "ُ": 96, | |
| "ِ": 97, | |
| "ّ": 98, | |
| "ْ": 99 | |
| }, | |
| "merges": [] | |
| } | |
| } |