{ "add_bos_token": true, "add_prefix_space": false, "added_tokens_decoder": { "1": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "50265": { "content": " ku", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50266": { "content": " za", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50267": { "content": "yar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50268": { "content": " wan", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50269": { "content": "sar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50270": { "content": "’", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50271": { "content": " yi", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50272": { "content": " kuma", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50273": { "content": "ikin", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50274": { "content": " yan", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50275": { "content": " shi", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50276": { "content": "uwa", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50277": { "content": "yin", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50278": { "content": " cikin", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50279": { "content": " kar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50280": { "content": "ƙ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50281": { "content": " cewa", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50282": { "content": " daga", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50283": { "content": " suka", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50284": { "content": "iyar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50285": { "content": "wam", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50286": { "content": " mai", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50287": { "content": " ji", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50288": { "content": " wannan", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50289": { "content": " han", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50290": { "content": "dda", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50291": { "content": " wani", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50292": { "content": " sha", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50293": { "content": "ɗ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50294": { "content": "shin", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50295": { "content": "ugaban", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50296": { "content": " kasa", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50297": { "content": " kasar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50298": { "content": " ƙ", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50299": { "content": " ci", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50300": { "content": " bayan", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50301": { "content": "wamn", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50302": { "content": " wanda", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50303": { "content": " tsa", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50304": { "content": " masu", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50305": { "content": "anar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50306": { "content": "tsa", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50307": { "content": " iya", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50308": { "content": " mata", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50309": { "content": "jeriya", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50310": { "content": " yana", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50311": { "content": " gaba", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50312": { "content": " dai", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50313": { "content": " zuwa", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50314": { "content": " duk", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50315": { "content": " zai", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50316": { "content": " kai", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50317": { "content": " sai", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50318": { "content": " Najeriya", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50319": { "content": " jihar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50320": { "content": " gi", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50321": { "content": " tare", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50322": { "content": " shugaban", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50323": { "content": " wasu", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50324": { "content": " loka", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50325": { "content": " ‘", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50326": { "content": "wamnat", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50327": { "content": "anin", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50328": { "content": " yadda", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50329": { "content": " suna", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50330": { "content": "yyan", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50331": { "content": " kara", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50332": { "content": " mutan", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50333": { "content": " matsa", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50334": { "content": " “", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50335": { "content": "kwa", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50336": { "content": " haka", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50337": { "content": " abin", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50338": { "content": " tsar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50339": { "content": " wata", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50340": { "content": "yyana", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50341": { "content": " ake", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50342": { "content": " huk", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50343": { "content": " bayyana", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50344": { "content": " ranar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50345": { "content": " samu", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50346": { "content": "”", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50347": { "content": " kwa", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50348": { "content": " inda", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50349": { "content": " biy", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50350": { "content": " wadan", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50351": { "content": "shen", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50352": { "content": " shekar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50353": { "content": " gwamnat", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50354": { "content": "bba", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50355": { "content": "kon", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50356": { "content": " lokacin", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50357": { "content": " wajen", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50358": { "content": "iyyar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50359": { "content": " amf", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50360": { "content": " mutane", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50361": { "content": "kara", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50362": { "content": "ungiyar", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50363": { "content": " yin", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "50364": { "content": " kya", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false } }, "bos_token": "", "clean_up_tokenization_spaces": true, "eos_token": "", "errors": "replace", "model_max_length": 1000000000000000019884624838656, "pad_token": "", "tokenizer_class": "GPT2Tokenizer", "unk_token": "" }