nllb-200-distilled-600M-v1 / added_tokens.json
akirus's picture
Upload tokenizer
b35e0a3 verified
{
"<mask>": 270447,
"ace_Arab": 270448,
"ace_Latn": 270449,
"acm_Arab": 270450,
"acq_Arab": 270451,
"aeb_Arab": 270452,
"afr_Latn": 270453,
"ajp_Arab": 270454,
"aka_Latn": 270455,
"als_Latn": 270456,
"amh_Ethi": 270457,
"apc_Arab": 270458,
"arb_Arab": 270459,
"ars_Arab": 270460,
"ary_Arab": 270461,
"arz_Arab": 270462,
"asm_Beng": 270463,
"ast_Latn": 270464,
"awa_Deva": 270465,
"ayr_Latn": 270466,
"azb_Arab": 270467,
"azj_Latn": 270468,
"bak_Cyrl": 270469,
"bam_Latn": 270470,
"ban_Latn": 270471,
"bel_Cyrl": 270472,
"bem_Latn": 270473,
"ben_Beng": 270474,
"bho_Deva": 270475,
"bjn_Arab": 270476,
"bjn_Latn": 270477,
"bod_Tibt": 270478,
"bos_Latn": 270479,
"bug_Latn": 270480,
"bul_Cyrl": 270481,
"cat_Latn": 270482,
"ceb_Latn": 270483,
"ces_Latn": 270484,
"cjk_Latn": 270485,
"ckb_Arab": 270486,
"crh_Latn": 270487,
"cym_Latn": 270488,
"dan_Latn": 270489,
"deu_Latn": 270490,
"dik_Latn": 270491,
"dyu_Latn": 270492,
"dzo_Tibt": 270493,
"ell_Grek": 270494,
"eng_Latn": 270495,
"epo_Latn": 270496,
"est_Latn": 270497,
"eus_Latn": 270498,
"ewe_Latn": 270499,
"fao_Latn": 270500,
"fij_Latn": 270501,
"fin_Latn": 270502,
"fon_Latn": 270503,
"fra_Latn": 270504,
"fur_Latn": 270505,
"fuv_Latn": 270506,
"gaz_Latn": 270507,
"gla_Latn": 270508,
"gle_Latn": 270509,
"glg_Latn": 270510,
"grn_Latn": 270511,
"guj_Gujr": 270512,
"hat_Latn": 270513,
"hau_Latn": 270514,
"heb_Hebr": 270515,
"hin_Deva": 270516,
"hne_Deva": 270517,
"hrv_Latn": 270518,
"hun_Latn": 270519,
"hye_Armn": 270520,
"ibo_Latn": 270521,
"ilo_Latn": 270522,
"ind_Latn": 270523,
"isl_Latn": 270524,
"ita_Latn": 270525,
"jav_Latn": 270526,
"jpn_Jpan": 270527,
"kab_Latn": 270528,
"kac_Latn": 270529,
"kam_Latn": 270530,
"kan_Knda": 270531,
"kas_Arab": 270532,
"kas_Deva": 270533,
"kat_Geor": 270534,
"kaz_Cyrl": 270535,
"kbp_Latn": 270536,
"kea_Latn": 270537,
"khk_Cyrl": 270538,
"khm_Khmr": 270539,
"kik_Latn": 270540,
"kin_Latn": 270541,
"kir_Cyrl": 270542,
"kmb_Latn": 270543,
"kmr_Latn": 270544,
"knc_Arab": 270545,
"knc_Latn": 270546,
"kon_Latn": 270547,
"kor_Hang": 270548,
"lao_Laoo": 270549,
"lez_Cyrl": 270550,
"lij_Latn": 270551,
"lim_Latn": 270552,
"lin_Latn": 270553,
"lit_Latn": 270554,
"lmo_Latn": 270555,
"ltg_Latn": 270556,
"ltz_Latn": 270557,
"lua_Latn": 270558,
"lug_Latn": 270559,
"luo_Latn": 270560,
"lus_Latn": 270561,
"lvs_Latn": 270562,
"mag_Deva": 270563,
"mai_Deva": 270564,
"mal_Mlym": 270565,
"mar_Deva": 270566,
"min_Latn": 270567,
"mkd_Cyrl": 270568,
"mlt_Latn": 270569,
"mni_Beng": 270570,
"mos_Latn": 270571,
"mri_Latn": 270572,
"mya_Mymr": 270573,
"nld_Latn": 270574,
"nno_Latn": 270575,
"nob_Latn": 270576,
"npi_Deva": 270577,
"nso_Latn": 270578,
"nus_Latn": 270579,
"nya_Latn": 270580,
"oci_Latn": 270581,
"ory_Orya": 270582,
"pag_Latn": 270583,
"pan_Guru": 270584,
"pap_Latn": 270585,
"pbt_Arab": 270586,
"pes_Arab": 270587,
"plt_Latn": 270588,
"pol_Latn": 270589,
"por_Latn": 270590,
"prs_Arab": 270591,
"quy_Latn": 270592,
"ron_Latn": 270593,
"run_Latn": 270594,
"rus_Cyrl": 270595,
"sag_Latn": 270596,
"san_Deva": 270597,
"sat_Beng": 270598,
"scn_Latn": 270599,
"shn_Mymr": 270600,
"sin_Sinh": 270601,
"slk_Latn": 270602,
"slv_Latn": 270603,
"smo_Latn": 270604,
"sna_Latn": 270605,
"snd_Arab": 270606,
"som_Latn": 270607,
"sot_Latn": 270608,
"spa_Latn": 270609,
"srd_Latn": 270610,
"srp_Cyrl": 270611,
"ssw_Latn": 270612,
"sun_Latn": 270613,
"swe_Latn": 270614,
"swh_Latn": 270615,
"szl_Latn": 270616,
"tam_Taml": 270617,
"taq_Latn": 270618,
"taq_Tfng": 270619,
"tat_Cyrl": 270620,
"tel_Telu": 270621,
"tgk_Cyrl": 270622,
"tgl_Latn": 270623,
"tha_Thai": 270624,
"tir_Ethi": 270625,
"tpi_Latn": 270626,
"tsn_Latn": 270627,
"tso_Latn": 270628,
"tuk_Latn": 270629,
"tum_Latn": 270630,
"tur_Latn": 270631,
"twi_Latn": 270632,
"tzm_Tfng": 270633,
"uig_Arab": 270634,
"ukr_Cyrl": 270635,
"umb_Latn": 270636,
"urd_Arab": 270637,
"uzn_Latn": 270638,
"vec_Latn": 270639,
"vie_Latn": 270640,
"war_Latn": 270641,
"wol_Latn": 270642,
"xho_Latn": 270643,
"ydd_Hebr": 270644,
"yor_Latn": 270645,
"yue_Hant": 270646,
"zho_Hans": 270647,
"zho_Hant": 270648,
"zsm_Latn": 270649,
"zul_Latn": 270650
}