low-resource-language-model-adaptation
/
adapted_tokenizers
/added-opt-eng
/opt_100-add_full-eng-opt
/tokenizer_config.json
{ | |
"add_bos_token": true, | |
"add_prefix_space": false, | |
"added_tokens_decoder": { | |
"1": { | |
"content": "<pad>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"2": { | |
"content": "</s>", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"50265": { | |
"content": " pla", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50266": { | |
"content": "ubl", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50267": { | |
"content": "ely", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50268": { | |
"content": "ilm", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50269": { | |
"content": "cted", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50270": { | |
"content": "arly", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50271": { | |
"content": " publ", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50272": { | |
"content": "overn", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50273": { | |
"content": "usic", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50274": { | |
"content": "ootball", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50275": { | |
"content": "istory", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50276": { | |
"content": "areer", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50277": { | |
"content": "stit", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50278": { | |
"content": "overnment", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50279": { | |
"content": "arri", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50280": { | |
"content": "owever", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50281": { | |
"content": "fess", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50282": { | |
"content": "ugust", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50283": { | |
"content": "ternational", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50284": { | |
"content": "tend", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50285": { | |
"content": "eptember", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50286": { | |
"content": "ctober", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50287": { | |
"content": " dif", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50288": { | |
"content": "pril", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50289": { | |
"content": " villa", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50290": { | |
"content": "’", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50291": { | |
"content": "ampionship", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50292": { | |
"content": "duced", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50293": { | |
"content": "ebru", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50294": { | |
"content": "ebruary", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50295": { | |
"content": "embers", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50296": { | |
"content": " poin", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50297": { | |
"content": "ivision", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50298": { | |
"content": "lack", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50299": { | |
"content": "ounc", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50300": { | |
"content": " Geor", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50301": { | |
"content": "arried", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50302": { | |
"content": "ouncil", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50303": { | |
"content": " consid", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50304": { | |
"content": "tended", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50305": { | |
"content": "loy", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50306": { | |
"content": " “", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50307": { | |
"content": "”", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50308": { | |
"content": " constru", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50309": { | |
"content": "eneral", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50310": { | |
"content": " Oly", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50311": { | |
"content": "adio", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50312": { | |
"content": "ecut", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50313": { | |
"content": " appoin", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50314": { | |
"content": " repla", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50315": { | |
"content": "ociety", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50316": { | |
"content": "forman", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50317": { | |
"content": "iforn", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50318": { | |
"content": "ifornia", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50319": { | |
"content": "stitute", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50320": { | |
"content": " emp", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50321": { | |
"content": "etwork", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50322": { | |
"content": " performan", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50323": { | |
"content": " examp", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50324": { | |
"content": " gradu", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50325": { | |
"content": " daugh", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50326": { | |
"content": "arliam", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50327": { | |
"content": " nov", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50328": { | |
"content": "arliament", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50329": { | |
"content": "gether", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50330": { | |
"content": " produc", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50331": { | |
"content": " Academ", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50332": { | |
"content": "overnor", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50333": { | |
"content": "ecutive", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50334": { | |
"content": " tourn", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50335": { | |
"content": " becom", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50336": { | |
"content": " earl", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50337": { | |
"content": "embly", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50338": { | |
"content": "eck", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50339": { | |
"content": " sele", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50340": { | |
"content": "eep", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50341": { | |
"content": "thlet", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50342": { | |
"content": "fic", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50343": { | |
"content": "reland", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50344": { | |
"content": "emor", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50345": { | |
"content": "ularly", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50346": { | |
"content": " colla", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50347": { | |
"content": "incip", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50348": { | |
"content": "ollowing", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50349": { | |
"content": "eleb", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50350": { | |
"content": " Pla", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50351": { | |
"content": " Legit", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50352": { | |
"content": "oviet", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50353": { | |
"content": " Histor", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50354": { | |
"content": "retary", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50355": { | |
"content": " nomin", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50356": { | |
"content": "lant", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50357": { | |
"content": "zil", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50358": { | |
"content": " Profess", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50359": { | |
"content": " plann", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50360": { | |
"content": " Emp", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50361": { | |
"content": "stitut", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50362": { | |
"content": " Biography", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50363": { | |
"content": " anc", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50364": { | |
"content": "ecess", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
} | |
}, | |
"bos_token": "</s>", | |
"clean_up_tokenization_spaces": true, | |
"eos_token": "</s>", | |
"errors": "replace", | |
"model_max_length": 1000000000000000019884624838656, | |
"pad_token": "<pad>", | |
"tokenizer_class": "GPT2Tokenizer", | |
"unk_token": "</s>" | |
} | |