Upload tokenizer
b35e0a3
verified
|
{ |
|
"<mask>": 270447, |
|
"ace_Arab": 270448, |
|
"ace_Latn": 270449, |
|
"acm_Arab": 270450, |
|
"acq_Arab": 270451, |
|
"aeb_Arab": 270452, |
|
"afr_Latn": 270453, |
|
"ajp_Arab": 270454, |
|
"aka_Latn": 270455, |
|
"als_Latn": 270456, |
|
"amh_Ethi": 270457, |
|
"apc_Arab": 270458, |
|
"arb_Arab": 270459, |
|
"ars_Arab": 270460, |
|
"ary_Arab": 270461, |
|
"arz_Arab": 270462, |
|
"asm_Beng": 270463, |
|
"ast_Latn": 270464, |
|
"awa_Deva": 270465, |
|
"ayr_Latn": 270466, |
|
"azb_Arab": 270467, |
|
"azj_Latn": 270468, |
|
"bak_Cyrl": 270469, |
|
"bam_Latn": 270470, |
|
"ban_Latn": 270471, |
|
"bel_Cyrl": 270472, |
|
"bem_Latn": 270473, |
|
"ben_Beng": 270474, |
|
"bho_Deva": 270475, |
|
"bjn_Arab": 270476, |
|
"bjn_Latn": 270477, |
|
"bod_Tibt": 270478, |
|
"bos_Latn": 270479, |
|
"bug_Latn": 270480, |
|
"bul_Cyrl": 270481, |
|
"cat_Latn": 270482, |
|
"ceb_Latn": 270483, |
|
"ces_Latn": 270484, |
|
"cjk_Latn": 270485, |
|
"ckb_Arab": 270486, |
|
"crh_Latn": 270487, |
|
"cym_Latn": 270488, |
|
"dan_Latn": 270489, |
|
"deu_Latn": 270490, |
|
"dik_Latn": 270491, |
|
"dyu_Latn": 270492, |
|
"dzo_Tibt": 270493, |
|
"ell_Grek": 270494, |
|
"eng_Latn": 270495, |
|
"epo_Latn": 270496, |
|
"est_Latn": 270497, |
|
"eus_Latn": 270498, |
|
"ewe_Latn": 270499, |
|
"fao_Latn": 270500, |
|
"fij_Latn": 270501, |
|
"fin_Latn": 270502, |
|
"fon_Latn": 270503, |
|
"fra_Latn": 270504, |
|
"fur_Latn": 270505, |
|
"fuv_Latn": 270506, |
|
"gaz_Latn": 270507, |
|
"gla_Latn": 270508, |
|
"gle_Latn": 270509, |
|
"glg_Latn": 270510, |
|
"grn_Latn": 270511, |
|
"guj_Gujr": 270512, |
|
"hat_Latn": 270513, |
|
"hau_Latn": 270514, |
|
"heb_Hebr": 270515, |
|
"hin_Deva": 270516, |
|
"hne_Deva": 270517, |
|
"hrv_Latn": 270518, |
|
"hun_Latn": 270519, |
|
"hye_Armn": 270520, |
|
"ibo_Latn": 270521, |
|
"ilo_Latn": 270522, |
|
"ind_Latn": 270523, |
|
"isl_Latn": 270524, |
|
"ita_Latn": 270525, |
|
"jav_Latn": 270526, |
|
"jpn_Jpan": 270527, |
|
"kab_Latn": 270528, |
|
"kac_Latn": 270529, |
|
"kam_Latn": 270530, |
|
"kan_Knda": 270531, |
|
"kas_Arab": 270532, |
|
"kas_Deva": 270533, |
|
"kat_Geor": 270534, |
|
"kaz_Cyrl": 270535, |
|
"kbp_Latn": 270536, |
|
"kea_Latn": 270537, |
|
"khk_Cyrl": 270538, |
|
"khm_Khmr": 270539, |
|
"kik_Latn": 270540, |
|
"kin_Latn": 270541, |
|
"kir_Cyrl": 270542, |
|
"kmb_Latn": 270543, |
|
"kmr_Latn": 270544, |
|
"knc_Arab": 270545, |
|
"knc_Latn": 270546, |
|
"kon_Latn": 270547, |
|
"kor_Hang": 270548, |
|
"lao_Laoo": 270549, |
|
"lez_Cyrl": 270550, |
|
"lij_Latn": 270551, |
|
"lim_Latn": 270552, |
|
"lin_Latn": 270553, |
|
"lit_Latn": 270554, |
|
"lmo_Latn": 270555, |
|
"ltg_Latn": 270556, |
|
"ltz_Latn": 270557, |
|
"lua_Latn": 270558, |
|
"lug_Latn": 270559, |
|
"luo_Latn": 270560, |
|
"lus_Latn": 270561, |
|
"lvs_Latn": 270562, |
|
"mag_Deva": 270563, |
|
"mai_Deva": 270564, |
|
"mal_Mlym": 270565, |
|
"mar_Deva": 270566, |
|
"min_Latn": 270567, |
|
"mkd_Cyrl": 270568, |
|
"mlt_Latn": 270569, |
|
"mni_Beng": 270570, |
|
"mos_Latn": 270571, |
|
"mri_Latn": 270572, |
|
"mya_Mymr": 270573, |
|
"nld_Latn": 270574, |
|
"nno_Latn": 270575, |
|
"nob_Latn": 270576, |
|
"npi_Deva": 270577, |
|
"nso_Latn": 270578, |
|
"nus_Latn": 270579, |
|
"nya_Latn": 270580, |
|
"oci_Latn": 270581, |
|
"ory_Orya": 270582, |
|
"pag_Latn": 270583, |
|
"pan_Guru": 270584, |
|
"pap_Latn": 270585, |
|
"pbt_Arab": 270586, |
|
"pes_Arab": 270587, |
|
"plt_Latn": 270588, |
|
"pol_Latn": 270589, |
|
"por_Latn": 270590, |
|
"prs_Arab": 270591, |
|
"quy_Latn": 270592, |
|
"ron_Latn": 270593, |
|
"run_Latn": 270594, |
|
"rus_Cyrl": 270595, |
|
"sag_Latn": 270596, |
|
"san_Deva": 270597, |
|
"sat_Beng": 270598, |
|
"scn_Latn": 270599, |
|
"shn_Mymr": 270600, |
|
"sin_Sinh": 270601, |
|
"slk_Latn": 270602, |
|
"slv_Latn": 270603, |
|
"smo_Latn": 270604, |
|
"sna_Latn": 270605, |
|
"snd_Arab": 270606, |
|
"som_Latn": 270607, |
|
"sot_Latn": 270608, |
|
"spa_Latn": 270609, |
|
"srd_Latn": 270610, |
|
"srp_Cyrl": 270611, |
|
"ssw_Latn": 270612, |
|
"sun_Latn": 270613, |
|
"swe_Latn": 270614, |
|
"swh_Latn": 270615, |
|
"szl_Latn": 270616, |
|
"tam_Taml": 270617, |
|
"taq_Latn": 270618, |
|
"taq_Tfng": 270619, |
|
"tat_Cyrl": 270620, |
|
"tel_Telu": 270621, |
|
"tgk_Cyrl": 270622, |
|
"tgl_Latn": 270623, |
|
"tha_Thai": 270624, |
|
"tir_Ethi": 270625, |
|
"tpi_Latn": 270626, |
|
"tsn_Latn": 270627, |
|
"tso_Latn": 270628, |
|
"tuk_Latn": 270629, |
|
"tum_Latn": 270630, |
|
"tur_Latn": 270631, |
|
"twi_Latn": 270632, |
|
"tzm_Tfng": 270633, |
|
"uig_Arab": 270634, |
|
"ukr_Cyrl": 270635, |
|
"umb_Latn": 270636, |
|
"urd_Arab": 270637, |
|
"uzn_Latn": 270638, |
|
"vec_Latn": 270639, |
|
"vie_Latn": 270640, |
|
"war_Latn": 270641, |
|
"wol_Latn": 270642, |
|
"xho_Latn": 270643, |
|
"ydd_Hebr": 270644, |
|
"yor_Latn": 270645, |
|
"yue_Hant": 270646, |
|
"zho_Hans": 270647, |
|
"zho_Hant": 270648, |
|
"zsm_Latn": 270649, |
|
"zul_Latn": 270650 |
|
} |
|
|