low-resource-language-model-adaptation
/
adapted_tokenizers
/added-opt-hau
/opt_100-add_full-hau-opt
/added_tokens.json
{ | |
" Najeriya": 50318, | |
" abin": 50337, | |
" ake": 50341, | |
" amf": 50359, | |
" bayan": 50300, | |
" bayyana": 50343, | |
" biy": 50349, | |
" cewa": 50281, | |
" ci": 50299, | |
" cikin": 50278, | |
" daga": 50282, | |
" dai": 50312, | |
" duk": 50314, | |
" gaba": 50311, | |
" gi": 50320, | |
" gwamnat": 50353, | |
" haka": 50336, | |
" han": 50289, | |
" huk": 50342, | |
" inda": 50348, | |
" iya": 50307, | |
" ji": 50287, | |
" jihar": 50319, | |
" kai": 50316, | |
" kar": 50279, | |
" kara": 50331, | |
" kasa": 50296, | |
" kasar": 50297, | |
" ku": 50265, | |
" kuma": 50272, | |
" kwa": 50347, | |
" kya": 50364, | |
" loka": 50324, | |
" lokacin": 50356, | |
" mai": 50286, | |
" masu": 50304, | |
" mata": 50308, | |
" matsa": 50333, | |
" mutan": 50332, | |
" mutane": 50360, | |
" ranar": 50344, | |
" sai": 50317, | |
" samu": 50345, | |
" sha": 50292, | |
" shekar": 50352, | |
" shi": 50275, | |
" shugaban": 50322, | |
" suka": 50283, | |
" suna": 50329, | |
" tare": 50321, | |
" tsa": 50303, | |
" tsar": 50338, | |
" wadan": 50350, | |
" wajen": 50357, | |
" wan": 50268, | |
" wanda": 50302, | |
" wani": 50291, | |
" wannan": 50288, | |
" wasu": 50323, | |
" wata": 50339, | |
" yadda": 50328, | |
" yan": 50274, | |
" yana": 50310, | |
" yi": 50271, | |
" yin": 50363, | |
" za": 50266, | |
" zai": 50315, | |
" zuwa": 50313, | |
" Ζ": 50298, | |
" β": 50325, | |
" β": 50334, | |
"anar": 50305, | |
"anin": 50327, | |
"bba": 50354, | |
"dda": 50290, | |
"ikin": 50273, | |
"iyar": 50284, | |
"iyyar": 50358, | |
"jeriya": 50309, | |
"kara": 50361, | |
"kon": 50355, | |
"kwa": 50335, | |
"sar": 50269, | |
"shen": 50351, | |
"shin": 50294, | |
"tsa": 50306, | |
"ugaban": 50295, | |
"ungiyar": 50362, | |
"uwa": 50276, | |
"wam": 50285, | |
"wamn": 50301, | |
"wamnat": 50326, | |
"yar": 50267, | |
"yin": 50277, | |
"yyan": 50330, | |
"yyana": 50340, | |
"Ζ": 50280, | |
"Ι": 50293, | |
"β": 50270, | |
"β": 50346 | |
} | |