Hugging Face's logo

Zaid
/
ashaar_model

Text Generation
Transformers
PyTorch
gpt2
text-generation-inference
Inference Endpoints
Model card Files Community
ashaar_model / tokenizer.json
Zaid's picture
Zaid
Training in progress, step 500
8ac169d
raw
history blame contribute delete
12.6 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|psep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "</|psep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<|vsep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "<|bsep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 5,
"content": "</|bsep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 6,
"content": "<|pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 7,
"content": "<|meter_0|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 8,
"content": "<|meter_1|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 9,
"content": "<|meter_2|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 10,
"content": "<|meter_3|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 11,
"content": "<|meter_4|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 12,
"content": "<|meter_5|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 13,
"content": "<|meter_6|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 14,
"content": "<|meter_7|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 15,
"content": "<|meter_8|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 16,
"content": "<|meter_9|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 17,
"content": "<|meter_10|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 18,
"content": "<|meter_11|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 19,
"content": "<|meter_12|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 20,
"content": "<|meter_13|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 21,
"content": "<|meter_14|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 22,
"content": "<|meter_15|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 23,
"content": "<|meter_16|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 24,
"content": "<|theme_0|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 25,
"content": "<|theme_1|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 26,
"content": "<|theme_2|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 27,
"content": "<|theme_3|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 28,
"content": "<|theme_4|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 29,
"content": "<|theme_5|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 30,
"content": "<|theme_6|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 31,
"content": "<|theme_7|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 32,
"content": "<|theme_8|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 33,
"content": "<|theme_9|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 34,
"content": "<|theme_10|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 35,
"content": "<|theme_11|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 36,
"content": "<|theme_12|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 37,
"content": "<|theme_13|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 38,
"content": "<|theme_14|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 39,
"content": "<|theme_15|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 40,
"content": "<|theme_16|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 41,
"content": "<|theme_17|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 42,
"content": "<|theme_18|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 43,
"content": "<|res_0|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 44,
"content": "<|res_1|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 45,
"content": "<|res_2|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 46,
"content": "<|res_3|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 47,
"content": "<|res_4|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 48,
"content": "<|res_5|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 49,
"content": "<|res_6|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 50,
"content": "<|res_7|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 51,
"content": "<|res_8|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 52,
"content": "<|res_9|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "UnicodeScripts"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"vocab": {
"<|endoftext|>": 0,
"<|psep|>": 1,
"</|psep|>": 2,
"<|vsep|>": 3,
"<|bsep|>": 4,
"</|bsep|>": 5,
"<|pad|>": 6,
"<|meter_0|>": 7,
"<|meter_1|>": 8,
"<|meter_2|>": 9,
"<|meter_3|>": 10,
"<|meter_4|>": 11,
"<|meter_5|>": 12,
"<|meter_6|>": 13,
"<|meter_7|>": 14,
"<|meter_8|>": 15,
"<|meter_9|>": 16,
"<|meter_10|>": 17,
"<|meter_11|>": 18,
"<|meter_12|>": 19,
"<|meter_13|>": 20,
"<|meter_14|>": 21,
"<|meter_15|>": 22,
"<|meter_16|>": 23,
"<|theme_0|>": 24,
"<|theme_1|>": 25,
"<|theme_2|>": 26,
"<|theme_3|>": 27,
"<|theme_4|>": 28,
"<|theme_5|>": 29,
"<|theme_6|>": 30,
"<|theme_7|>": 31,
"<|theme_8|>": 32,
"<|theme_9|>": 33,
"<|theme_10|>": 34,
"<|theme_11|>": 35,
"<|theme_12|>": 36,
"<|theme_13|>": 37,
"<|theme_14|>": 38,
"<|theme_15|>": 39,
"<|theme_16|>": 40,
"<|theme_17|>": 41,
"<|theme_18|>": 42,
"<|res_0|>": 43,
"<|res_1|>": 44,
"<|res_2|>": 45,
"<|res_3|>": 46,
"<|res_4|>": 47,
"<|res_5|>": 48,
"<|res_6|>": 49,
"<|res_7|>": 50,
"<|res_8|>": 51,
"<|res_9|>": 52,
" ": 53,
"/": 54,
"0": 55,
"1": 56,
"2": 57,
"3": 58,
"4": 59,
"5": 60,
"6": 61,
"7": 62,
"8": 63,
"9": 64,
"<": 65,
">": 66,
"_": 67,
"b": 68,
"e": 69,
"h": 70,
"m": 71,
"p": 72,
"r": 73,
"s": 74,
"t": 75,
"v": 76,
"|": 77,
"~": 78,
"ء": 79,
"أ": 80,
"ؤ": 81,
"ئ": 82,
"ا": 83,
"ب": 84,
"ة": 85,
"ت": 86,
"ث": 87,
"ج": 88,
"ح": 89,
"خ": 90,
"د": 91,
"ذ": 92,
"ر": 93,
"ز": 94,
"س": 95,
"ش": 96,
"ص": 97,
"ض": 98,
"ط": 99,
"ظ": 100,
"ع": 101,
"غ": 102,
"ف": 103,
"ق": 104,
"ك": 105,
"ل": 106,
"م": 107,
"ن": 108,
"ه": 109,
"و": 110,
"ى": 111,
"ي": 112,
"ً": 113,
"ٌ": 114,
"ٍ": 115,
"َ": 116,
"ُ": 117,
"ِ": 118,
"ّ": 119,
"ْ": 120
},
"merges": []
}
}