Upload tokenizer
ae10b6f
verified
|
{ |
|
"<mask>": 268324, |
|
"ace_Arab": 268325, |
|
"ace_Latn": 268326, |
|
"acm_Arab": 268327, |
|
"acq_Arab": 268328, |
|
"aeb_Arab": 268329, |
|
"afr_Latn": 268330, |
|
"ajp_Arab": 268331, |
|
"aka_Latn": 268332, |
|
"als_Latn": 268333, |
|
"amh_Ethi": 268334, |
|
"apc_Arab": 268335, |
|
"arb_Arab": 268336, |
|
"ars_Arab": 268337, |
|
"ary_Arab": 268338, |
|
"arz_Arab": 268339, |
|
"asm_Beng": 268340, |
|
"ast_Latn": 268341, |
|
"awa_Deva": 268342, |
|
"ayr_Latn": 268343, |
|
"azb_Arab": 268344, |
|
"azj_Latn": 268345, |
|
"bak_Cyrl": 268346, |
|
"bam_Latn": 268347, |
|
"ban_Latn": 268348, |
|
"bel_Cyrl": 268349, |
|
"bem_Latn": 268350, |
|
"ben_Beng": 268351, |
|
"bho_Deva": 268352, |
|
"bjn_Arab": 268353, |
|
"bjn_Latn": 268354, |
|
"bod_Tibt": 268355, |
|
"bos_Latn": 268356, |
|
"bug_Latn": 268357, |
|
"bul_Cyrl": 268358, |
|
"cat_Latn": 268359, |
|
"ceb_Latn": 268360, |
|
"ces_Latn": 268361, |
|
"cjk_Latn": 268362, |
|
"ckb_Arab": 268363, |
|
"crh_Latn": 268364, |
|
"cym_Latn": 268365, |
|
"dan_Latn": 268366, |
|
"deu_Latn": 268367, |
|
"dik_Latn": 268368, |
|
"dyu_Latn": 268369, |
|
"dzo_Tibt": 268370, |
|
"ell_Grek": 268371, |
|
"eng_Latn": 268372, |
|
"epo_Latn": 268373, |
|
"est_Latn": 268374, |
|
"eus_Latn": 268375, |
|
"ewe_Latn": 268376, |
|
"fao_Latn": 268377, |
|
"fij_Latn": 268378, |
|
"fin_Latn": 268379, |
|
"fon_Latn": 268380, |
|
"fra_Latn": 268381, |
|
"fur_Latn": 268382, |
|
"fuv_Latn": 268383, |
|
"gaz_Latn": 268384, |
|
"gla_Latn": 268385, |
|
"gle_Latn": 268386, |
|
"glg_Latn": 268387, |
|
"grn_Latn": 268388, |
|
"guj_Gujr": 268389, |
|
"hat_Latn": 268390, |
|
"hau_Latn": 268391, |
|
"heb_Hebr": 268392, |
|
"hin_Deva": 268393, |
|
"hne_Deva": 268394, |
|
"hrv_Latn": 268395, |
|
"hun_Latn": 268396, |
|
"hye_Armn": 268397, |
|
"ibo_Latn": 268398, |
|
"ilo_Latn": 268399, |
|
"ind_Latn": 268400, |
|
"isl_Latn": 268401, |
|
"ita_Latn": 268402, |
|
"jav_Latn": 268403, |
|
"jpn_Jpan": 268404, |
|
"kab_Latn": 268405, |
|
"kac_Latn": 268406, |
|
"kam_Latn": 268407, |
|
"kan_Knda": 268408, |
|
"kas_Arab": 268409, |
|
"kas_Deva": 268410, |
|
"kat_Geor": 268411, |
|
"kaz_Cyrl": 268412, |
|
"kbp_Latn": 268413, |
|
"kea_Latn": 268414, |
|
"khk_Cyrl": 268415, |
|
"khm_Khmr": 268416, |
|
"kik_Latn": 268417, |
|
"kin_Latn": 268418, |
|
"kir_Cyrl": 268419, |
|
"kmb_Latn": 268420, |
|
"kmr_Latn": 268421, |
|
"knc_Arab": 268422, |
|
"knc_Latn": 268423, |
|
"kon_Latn": 268424, |
|
"kor_Hang": 268425, |
|
"lao_Laoo": 268426, |
|
"lez_Cyrl": 268427, |
|
"lij_Latn": 268428, |
|
"lim_Latn": 268429, |
|
"lin_Latn": 268430, |
|
"lit_Latn": 268431, |
|
"lmo_Latn": 268432, |
|
"ltg_Latn": 268433, |
|
"ltz_Latn": 268434, |
|
"lua_Latn": 268435, |
|
"lug_Latn": 268436, |
|
"luo_Latn": 268437, |
|
"lus_Latn": 268438, |
|
"lvs_Latn": 268439, |
|
"mag_Deva": 268440, |
|
"mai_Deva": 268441, |
|
"mal_Mlym": 268442, |
|
"mar_Deva": 268443, |
|
"min_Latn": 268444, |
|
"mkd_Cyrl": 268445, |
|
"mlt_Latn": 268446, |
|
"mni_Beng": 268447, |
|
"mos_Latn": 268448, |
|
"mri_Latn": 268449, |
|
"mya_Mymr": 268450, |
|
"nld_Latn": 268451, |
|
"nno_Latn": 268452, |
|
"nob_Latn": 268453, |
|
"npi_Deva": 268454, |
|
"nso_Latn": 268455, |
|
"nus_Latn": 268456, |
|
"nya_Latn": 268457, |
|
"oci_Latn": 268458, |
|
"ory_Orya": 268459, |
|
"pag_Latn": 268460, |
|
"pan_Guru": 268461, |
|
"pap_Latn": 268462, |
|
"pbt_Arab": 268463, |
|
"pes_Arab": 268464, |
|
"plt_Latn": 268465, |
|
"pol_Latn": 268466, |
|
"por_Latn": 268467, |
|
"prs_Arab": 268468, |
|
"quy_Latn": 268469, |
|
"ron_Latn": 268470, |
|
"run_Latn": 268471, |
|
"rus_Cyrl": 268472, |
|
"sag_Latn": 268473, |
|
"san_Deva": 268474, |
|
"sat_Beng": 268475, |
|
"scn_Latn": 268476, |
|
"shn_Mymr": 268477, |
|
"sin_Sinh": 268478, |
|
"slk_Latn": 268479, |
|
"slv_Latn": 268480, |
|
"smo_Latn": 268481, |
|
"sna_Latn": 268482, |
|
"snd_Arab": 268483, |
|
"som_Latn": 268484, |
|
"sot_Latn": 268485, |
|
"spa_Latn": 268486, |
|
"srd_Latn": 268487, |
|
"srp_Cyrl": 268488, |
|
"ssw_Latn": 268489, |
|
"sun_Latn": 268490, |
|
"swe_Latn": 268491, |
|
"swh_Latn": 268492, |
|
"szl_Latn": 268493, |
|
"tam_Taml": 268494, |
|
"taq_Latn": 268495, |
|
"taq_Tfng": 268496, |
|
"tat_Cyrl": 268497, |
|
"tel_Telu": 268498, |
|
"tgk_Cyrl": 268499, |
|
"tgl_Latn": 268500, |
|
"tha_Thai": 268501, |
|
"tir_Ethi": 268502, |
|
"tpi_Latn": 268503, |
|
"tsn_Latn": 268504, |
|
"tso_Latn": 268505, |
|
"tuk_Latn": 268506, |
|
"tum_Latn": 268507, |
|
"tur_Latn": 268508, |
|
"twi_Latn": 268509, |
|
"tzm_Tfng": 268510, |
|
"uig_Arab": 268511, |
|
"ukr_Cyrl": 268512, |
|
"umb_Latn": 268513, |
|
"urd_Arab": 268514, |
|
"uzn_Latn": 268515, |
|
"vec_Latn": 268516, |
|
"vie_Latn": 268517, |
|
"war_Latn": 268518, |
|
"wol_Latn": 268519, |
|
"xho_Latn": 268520, |
|
"ydd_Hebr": 268521, |
|
"yor_Latn": 268522, |
|
"yue_Hant": 268523, |
|
"zho_Hans": 268524, |
|
"zho_Hant": 268525, |
|
"zsm_Latn": 268526, |
|
"zul_Latn": 268527 |
|
} |
|
|