| { | |
| "added_tokens_decoder": { | |
| "0": { | |
| "content": "<s>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "1": { | |
| "content": "<pad>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "2": { | |
| "content": "</s>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "3": { | |
| "content": "<unk>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395270": { | |
| "content": "<mask>", | |
| "lstrip": true, | |
| "normalized": true, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395271": { | |
| "content": "ace_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395272": { | |
| "content": "ace_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395273": { | |
| "content": "acm_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395274": { | |
| "content": "acq_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395275": { | |
| "content": "aeb_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395276": { | |
| "content": "afr_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395277": { | |
| "content": "ajp_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395278": { | |
| "content": "aka_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395279": { | |
| "content": "als_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395280": { | |
| "content": "amh_Ethi", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395281": { | |
| "content": "apc_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395282": { | |
| "content": "arb_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395283": { | |
| "content": "ars_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395284": { | |
| "content": "ary_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395285": { | |
| "content": "arz_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395286": { | |
| "content": "asm_Beng", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395287": { | |
| "content": "ast_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395288": { | |
| "content": "awa_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395289": { | |
| "content": "ayr_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395290": { | |
| "content": "azb_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395291": { | |
| "content": "azj_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395292": { | |
| "content": "bak_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395293": { | |
| "content": "bam_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395294": { | |
| "content": "ban_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395295": { | |
| "content": "bel_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395296": { | |
| "content": "bem_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395297": { | |
| "content": "ben_Beng", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395298": { | |
| "content": "bho_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395299": { | |
| "content": "bjn_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395300": { | |
| "content": "bjn_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395301": { | |
| "content": "bod_Tibt", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395302": { | |
| "content": "bos_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395303": { | |
| "content": "bug_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395304": { | |
| "content": "bul_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395305": { | |
| "content": "bxr_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395306": { | |
| "content": "cat_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395307": { | |
| "content": "ceb_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395308": { | |
| "content": "ces_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395309": { | |
| "content": "che_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395310": { | |
| "content": "chv_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395311": { | |
| "content": "cjk_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395312": { | |
| "content": "ckb_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395313": { | |
| "content": "crh_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395314": { | |
| "content": "cym_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395315": { | |
| "content": "dan_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395316": { | |
| "content": "deu_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395317": { | |
| "content": "dik_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395318": { | |
| "content": "dyu_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395319": { | |
| "content": "dzo_Tibt", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395320": { | |
| "content": "ell_Grek", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395321": { | |
| "content": "eng_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395322": { | |
| "content": "epo_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395323": { | |
| "content": "est_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395324": { | |
| "content": "eus_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395325": { | |
| "content": "ewe_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395326": { | |
| "content": "fao_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395327": { | |
| "content": "fij_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395328": { | |
| "content": "fin_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395329": { | |
| "content": "fon_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395330": { | |
| "content": "fra_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395331": { | |
| "content": "fur_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395332": { | |
| "content": "fuv_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395333": { | |
| "content": "gaz_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395334": { | |
| "content": "gla_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395335": { | |
| "content": "gle_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395336": { | |
| "content": "glg_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395337": { | |
| "content": "grn_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395338": { | |
| "content": "guj_Gujr", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395339": { | |
| "content": "hat_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395340": { | |
| "content": "hau_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395341": { | |
| "content": "heb_Hebr", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395342": { | |
| "content": "hin_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395343": { | |
| "content": "hne_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395344": { | |
| "content": "hrv_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395345": { | |
| "content": "hun_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395346": { | |
| "content": "hye_Armn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395347": { | |
| "content": "ibo_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395348": { | |
| "content": "ilo_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395349": { | |
| "content": "ind_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395350": { | |
| "content": "isl_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395351": { | |
| "content": "ita_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395352": { | |
| "content": "jav_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395353": { | |
| "content": "jpn_Jpan", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395354": { | |
| "content": "kab_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395355": { | |
| "content": "kac_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395356": { | |
| "content": "kam_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395357": { | |
| "content": "kan_Knda", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395358": { | |
| "content": "kas_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395359": { | |
| "content": "kas_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395360": { | |
| "content": "kat_Geor", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395361": { | |
| "content": "kaz_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395362": { | |
| "content": "kbp_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395363": { | |
| "content": "kea_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395364": { | |
| "content": "khk_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395365": { | |
| "content": "khm_Khmr", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395366": { | |
| "content": "kik_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395367": { | |
| "content": "kin_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395368": { | |
| "content": "kir_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395369": { | |
| "content": "kjh_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395370": { | |
| "content": "kmb_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395371": { | |
| "content": "kmr_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395372": { | |
| "content": "knc_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395373": { | |
| "content": "knc_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395374": { | |
| "content": "kon_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395375": { | |
| "content": "kor_Hang", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395376": { | |
| "content": "krc_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395377": { | |
| "content": "lao_Laoo", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395378": { | |
| "content": "lij_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395379": { | |
| "content": "lim_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395380": { | |
| "content": "lin_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395381": { | |
| "content": "lit_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395382": { | |
| "content": "lmo_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395383": { | |
| "content": "ltg_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395384": { | |
| "content": "ltz_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395385": { | |
| "content": "lua_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395386": { | |
| "content": "lug_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395387": { | |
| "content": "luo_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395388": { | |
| "content": "lus_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395389": { | |
| "content": "lvs_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395390": { | |
| "content": "mag_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395391": { | |
| "content": "mai_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395392": { | |
| "content": "mal_Mlym", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395393": { | |
| "content": "mar_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395394": { | |
| "content": "mhr_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395395": { | |
| "content": "min_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395396": { | |
| "content": "mkd_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395397": { | |
| "content": "mlt_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395398": { | |
| "content": "mni_Beng", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395399": { | |
| "content": "mos_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395400": { | |
| "content": "mri_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395401": { | |
| "content": "mya_Mymr", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395402": { | |
| "content": "myv_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395403": { | |
| "content": "nld_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395404": { | |
| "content": "nno_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395405": { | |
| "content": "nob_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395406": { | |
| "content": "npi_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395407": { | |
| "content": "nso_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395408": { | |
| "content": "nus_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395409": { | |
| "content": "nya_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395410": { | |
| "content": "oci_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395411": { | |
| "content": "ory_Orya", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395412": { | |
| "content": "pag_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395413": { | |
| "content": "pan_Guru", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395414": { | |
| "content": "pap_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395415": { | |
| "content": "pbt_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395416": { | |
| "content": "pes_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395417": { | |
| "content": "plt_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395418": { | |
| "content": "pol_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395419": { | |
| "content": "por_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395420": { | |
| "content": "prs_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395421": { | |
| "content": "quy_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395422": { | |
| "content": "ron_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395423": { | |
| "content": "run_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395424": { | |
| "content": "rus_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395425": { | |
| "content": "sag_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395426": { | |
| "content": "san_Deva", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395427": { | |
| "content": "sat_Beng", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395428": { | |
| "content": "scn_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395429": { | |
| "content": "shn_Mymr", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395430": { | |
| "content": "sin_Sinh", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395431": { | |
| "content": "slk_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395432": { | |
| "content": "slv_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395433": { | |
| "content": "smo_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395434": { | |
| "content": "sna_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395435": { | |
| "content": "snd_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395436": { | |
| "content": "som_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395437": { | |
| "content": "sot_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395438": { | |
| "content": "spa_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395439": { | |
| "content": "srd_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395440": { | |
| "content": "srp_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395441": { | |
| "content": "ssw_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395442": { | |
| "content": "sun_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395443": { | |
| "content": "swe_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395444": { | |
| "content": "swh_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395445": { | |
| "content": "szl_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395446": { | |
| "content": "tam_Taml", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395447": { | |
| "content": "taq_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395448": { | |
| "content": "taq_Tfng", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395449": { | |
| "content": "tat_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395450": { | |
| "content": "tel_Telu", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395451": { | |
| "content": "tgk_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395452": { | |
| "content": "tgl_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395453": { | |
| "content": "tha_Thai", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395454": { | |
| "content": "tir_Ethi", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395455": { | |
| "content": "tpi_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395456": { | |
| "content": "tsn_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395457": { | |
| "content": "tso_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395458": { | |
| "content": "tuk_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395459": { | |
| "content": "tum_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395460": { | |
| "content": "tur_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395461": { | |
| "content": "twi_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395462": { | |
| "content": "tyv_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395463": { | |
| "content": "tzm_Tfng", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395464": { | |
| "content": "uig_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395465": { | |
| "content": "ukr_Cyrl", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395466": { | |
| "content": "umb_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395467": { | |
| "content": "urd_Arab", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395468": { | |
| "content": "uzn_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395469": { | |
| "content": "vec_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395470": { | |
| "content": "vie_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395471": { | |
| "content": "war_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395472": { | |
| "content": "wol_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395473": { | |
| "content": "xho_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395474": { | |
| "content": "ydd_Hebr", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395475": { | |
| "content": "yor_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395476": { | |
| "content": "yue_Hant", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395477": { | |
| "content": "zho_Hans", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395478": { | |
| "content": "zho_Hant", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395479": { | |
| "content": "zsm_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "395480": { | |
| "content": "zul_Latn", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| } | |
| }, | |
| "additional_special_tokens": [ | |
| "ace_Arab", | |
| "ace_Latn", | |
| "acm_Arab", | |
| "acq_Arab", | |
| "aeb_Arab", | |
| "afr_Latn", | |
| "ajp_Arab", | |
| "aka_Latn", | |
| "als_Latn", | |
| "amh_Ethi", | |
| "apc_Arab", | |
| "arb_Arab", | |
| "ars_Arab", | |
| "ary_Arab", | |
| "arz_Arab", | |
| "asm_Beng", | |
| "ast_Latn", | |
| "awa_Deva", | |
| "ayr_Latn", | |
| "azb_Arab", | |
| "azj_Latn", | |
| "bak_Cyrl", | |
| "bam_Latn", | |
| "ban_Latn", | |
| "bel_Cyrl", | |
| "bem_Latn", | |
| "ben_Beng", | |
| "bho_Deva", | |
| "bjn_Arab", | |
| "bjn_Latn", | |
| "bod_Tibt", | |
| "bos_Latn", | |
| "bug_Latn", | |
| "bul_Cyrl", | |
| "bxr_Cyrl", | |
| "cat_Latn", | |
| "ceb_Latn", | |
| "ces_Latn", | |
| "che_Cyrl", | |
| "chv_Cyrl", | |
| "cjk_Latn", | |
| "ckb_Arab", | |
| "crh_Latn", | |
| "cym_Latn", | |
| "dan_Latn", | |
| "deu_Latn", | |
| "dik_Latn", | |
| "dyu_Latn", | |
| "dzo_Tibt", | |
| "ell_Grek", | |
| "eng_Latn", | |
| "epo_Latn", | |
| "est_Latn", | |
| "eus_Latn", | |
| "ewe_Latn", | |
| "fao_Latn", | |
| "fij_Latn", | |
| "fin_Latn", | |
| "fon_Latn", | |
| "fra_Latn", | |
| "fur_Latn", | |
| "fuv_Latn", | |
| "gaz_Latn", | |
| "gla_Latn", | |
| "gle_Latn", | |
| "glg_Latn", | |
| "grn_Latn", | |
| "guj_Gujr", | |
| "hat_Latn", | |
| "hau_Latn", | |
| "heb_Hebr", | |
| "hin_Deva", | |
| "hne_Deva", | |
| "hrv_Latn", | |
| "hun_Latn", | |
| "hye_Armn", | |
| "ibo_Latn", | |
| "ilo_Latn", | |
| "ind_Latn", | |
| "isl_Latn", | |
| "ita_Latn", | |
| "jav_Latn", | |
| "jpn_Jpan", | |
| "kab_Latn", | |
| "kac_Latn", | |
| "kam_Latn", | |
| "kan_Knda", | |
| "kas_Arab", | |
| "kas_Deva", | |
| "kat_Geor", | |
| "kaz_Cyrl", | |
| "kbp_Latn", | |
| "kea_Latn", | |
| "khk_Cyrl", | |
| "khm_Khmr", | |
| "kik_Latn", | |
| "kin_Latn", | |
| "kir_Cyrl", | |
| "kjh_Cyrl", | |
| "kmb_Latn", | |
| "kmr_Latn", | |
| "knc_Arab", | |
| "knc_Latn", | |
| "kon_Latn", | |
| "kor_Hang", | |
| "krc_Cyrl", | |
| "lao_Laoo", | |
| "lij_Latn", | |
| "lim_Latn", | |
| "lin_Latn", | |
| "lit_Latn", | |
| "lmo_Latn", | |
| "ltg_Latn", | |
| "ltz_Latn", | |
| "lua_Latn", | |
| "lug_Latn", | |
| "luo_Latn", | |
| "lus_Latn", | |
| "lvs_Latn", | |
| "mag_Deva", | |
| "mai_Deva", | |
| "mal_Mlym", | |
| "mar_Deva", | |
| "mhr_Cyrl", | |
| "min_Latn", | |
| "mkd_Cyrl", | |
| "mlt_Latn", | |
| "mni_Beng", | |
| "mos_Latn", | |
| "mri_Latn", | |
| "mya_Mymr", | |
| "myv_Cyrl", | |
| "nld_Latn", | |
| "nno_Latn", | |
| "nob_Latn", | |
| "npi_Deva", | |
| "nso_Latn", | |
| "nus_Latn", | |
| "nya_Latn", | |
| "oci_Latn", | |
| "ory_Orya", | |
| "pag_Latn", | |
| "pan_Guru", | |
| "pap_Latn", | |
| "pbt_Arab", | |
| "pes_Arab", | |
| "plt_Latn", | |
| "pol_Latn", | |
| "por_Latn", | |
| "prs_Arab", | |
| "quy_Latn", | |
| "ron_Latn", | |
| "run_Latn", | |
| "rus_Cyrl", | |
| "sag_Latn", | |
| "san_Deva", | |
| "sat_Beng", | |
| "scn_Latn", | |
| "shn_Mymr", | |
| "sin_Sinh", | |
| "slk_Latn", | |
| "slv_Latn", | |
| "smo_Latn", | |
| "sna_Latn", | |
| "snd_Arab", | |
| "som_Latn", | |
| "sot_Latn", | |
| "spa_Latn", | |
| "srd_Latn", | |
| "srp_Cyrl", | |
| "ssw_Latn", | |
| "sun_Latn", | |
| "swe_Latn", | |
| "swh_Latn", | |
| "szl_Latn", | |
| "tam_Taml", | |
| "taq_Latn", | |
| "taq_Tfng", | |
| "tat_Cyrl", | |
| "tel_Telu", | |
| "tgk_Cyrl", | |
| "tgl_Latn", | |
| "tha_Thai", | |
| "tir_Ethi", | |
| "tpi_Latn", | |
| "tsn_Latn", | |
| "tso_Latn", | |
| "tuk_Latn", | |
| "tum_Latn", | |
| "tur_Latn", | |
| "twi_Latn", | |
| "tyv_Cyrl", | |
| "tzm_Tfng", | |
| "uig_Arab", | |
| "ukr_Cyrl", | |
| "umb_Latn", | |
| "urd_Arab", | |
| "uzn_Latn", | |
| "vec_Latn", | |
| "vie_Latn", | |
| "war_Latn", | |
| "wol_Latn", | |
| "xho_Latn", | |
| "ydd_Hebr", | |
| "yor_Latn", | |
| "yue_Hant", | |
| "zho_Hans", | |
| "zho_Hant", | |
| "zsm_Latn", | |
| "zul_Latn" | |
| ], | |
| "bos_token": "<s>", | |
| "clean_up_tokenization_spaces": true, | |
| "cls_token": "<s>", | |
| "eos_token": "</s>", | |
| "legacy_behaviour": false, | |
| "mask_token": "<mask>", | |
| "model_max_length": 1024, | |
| "pad_token": "<pad>", | |
| "sep_token": "</s>", | |
| "sp_model_kwargs": {}, | |
| "src_lang": "rus_Cyrl", | |
| "tgt_lang": "bak_Cyrl", | |
| "tokenizer_class": "NllbTokenizer", | |
| "unk_token": "<unk>" | |
| } | |