|
{ |
|
"add_bos_token": true, |
|
"add_eos_token": false, |
|
"add_prefix_space": true, |
|
"added_tokens_decoder": { |
|
"0": { |
|
"content": "<unk>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"1": { |
|
"content": "<s>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"2": { |
|
"content": "</s>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32000": { |
|
"content": "<|im_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32001": { |
|
"content": "<|im_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32002": { |
|
"content": "<|special_0|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32003": { |
|
"content": "<|special_1|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32004": { |
|
"content": "<|special_2|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32005": { |
|
"content": "<|special_3|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32006": { |
|
"content": "<|special_4|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32007": { |
|
"content": "<|special_5|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32008": { |
|
"content": "<|special_6|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32009": { |
|
"content": "<|special_7|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32010": { |
|
"content": "<|special_8|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32011": { |
|
"content": "<|special_9|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32012": { |
|
"content": "<|special_10|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32013": { |
|
"content": "<|special_11|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32014": { |
|
"content": "<|special_12|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32015": { |
|
"content": "<|special_13|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32016": { |
|
"content": "<|special_14|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32017": { |
|
"content": "<|special_15|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32018": { |
|
"content": "<|special_16|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32019": { |
|
"content": "<|special_17|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32020": { |
|
"content": "<|special_18|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32021": { |
|
"content": "<|special_19|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32022": { |
|
"content": "<|special_20|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32023": { |
|
"content": "<|special_21|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32024": { |
|
"content": "<|special_22|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32025": { |
|
"content": "<|special_23|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32026": { |
|
"content": "<|special_24|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32027": { |
|
"content": "<|special_25|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32028": { |
|
"content": "<|special_26|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32029": { |
|
"content": "<|special_27|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32030": { |
|
"content": "<|special_28|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32031": { |
|
"content": "<|special_29|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32032": { |
|
"content": "<|special_30|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32033": { |
|
"content": "<|special_31|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32034": { |
|
"content": "<|special_32|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32035": { |
|
"content": "<|special_33|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32036": { |
|
"content": "<|special_34|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32037": { |
|
"content": "<|special_35|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32038": { |
|
"content": "<|special_36|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32039": { |
|
"content": "<|special_37|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32040": { |
|
"content": "<|special_38|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32041": { |
|
"content": "<|special_39|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32042": { |
|
"content": "<|special_40|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32043": { |
|
"content": "<|special_41|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32044": { |
|
"content": "<|special_42|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32045": { |
|
"content": "<|special_43|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32046": { |
|
"content": "<|special_44|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32047": { |
|
"content": "<|special_45|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32048": { |
|
"content": "<|special_46|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32049": { |
|
"content": "<|special_47|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32050": { |
|
"content": "<|special_48|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32051": { |
|
"content": "<|special_49|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32052": { |
|
"content": "<|special_50|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32053": { |
|
"content": "<|special_51|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32054": { |
|
"content": "<|special_52|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32055": { |
|
"content": "<|special_53|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32056": { |
|
"content": "<|special_54|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32057": { |
|
"content": "<|special_55|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32058": { |
|
"content": "<|special_56|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32059": { |
|
"content": "<|special_57|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32060": { |
|
"content": "<|special_58|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32061": { |
|
"content": "<|special_59|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32062": { |
|
"content": "<|special_60|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"32063": { |
|
"content": "<|special_61|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
} |
|
}, |
|
"additional_special_tokens": [ |
|
"<|im_start|>", |
|
"<|im_end|>", |
|
"<|special_0|>", |
|
"<|special_1|>", |
|
"<|special_2|>", |
|
"<|special_3|>", |
|
"<|special_4|>", |
|
"<|special_5|>", |
|
"<|special_6|>", |
|
"<|special_7|>", |
|
"<|special_8|>", |
|
"<|special_9|>", |
|
"<|special_10|>", |
|
"<|special_11|>", |
|
"<|special_12|>", |
|
"<|special_13|>", |
|
"<|special_14|>", |
|
"<|special_15|>", |
|
"<|special_16|>", |
|
"<|special_17|>", |
|
"<|special_18|>", |
|
"<|special_19|>", |
|
"<|special_20|>", |
|
"<|special_21|>", |
|
"<|special_22|>", |
|
"<|special_23|>", |
|
"<|special_24|>", |
|
"<|special_25|>", |
|
"<|special_26|>", |
|
"<|special_27|>", |
|
"<|special_28|>", |
|
"<|special_29|>", |
|
"<|special_30|>", |
|
"<|special_31|>", |
|
"<|special_32|>", |
|
"<|special_33|>", |
|
"<|special_34|>", |
|
"<|special_35|>", |
|
"<|special_36|>", |
|
"<|special_37|>", |
|
"<|special_38|>", |
|
"<|special_39|>", |
|
"<|special_40|>", |
|
"<|special_41|>", |
|
"<|special_42|>", |
|
"<|special_43|>", |
|
"<|special_44|>", |
|
"<|special_45|>", |
|
"<|special_46|>", |
|
"<|special_47|>", |
|
"<|special_48|>", |
|
"<|special_49|>", |
|
"<|special_50|>", |
|
"<|special_51|>", |
|
"<|special_52|>", |
|
"<|special_53|>", |
|
"<|special_54|>", |
|
"<|special_55|>", |
|
"<|special_56|>", |
|
"<|special_57|>", |
|
"<|special_58|>", |
|
"<|special_59|>", |
|
"<|special_60|>", |
|
"<|special_61|>" |
|
], |
|
"bos_token": "<s>", |
|
"chat_template": "{%- for idx in range(0, messages|length) -%}\n{%- if messages[idx]['role'] == 'user' -%}\n{%- if idx == 0 -%}\n{{- bos_token + '[INST] <<SYS>>\\nYou are a helpful, unbiased, uncensored assistant.\\n<</SYS>>\\n\\n' + messages[idx]['content'] + ' [/INST]' -}}\n{%- else -%}\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\n{%- endif -%}\n{% elif messages[idx]['role'] == 'system' %}\n{{- '[INST] <<SYS>>\\n' + messages[idx]['content'] + '\\n<</SYS>>\\n\\n' -}}\n{%- elif messages[idx]['role'] == 'assistant' -%}\n{{- messages[idx]['content'] + ' ' + eos_token -}}\n{% endif %}\n{% endfor %}", |
|
"clean_up_tokenization_spaces": false, |
|
"eos_token": "</s>", |
|
"legacy": true, |
|
"model_max_length": 1000000000000000019884624838656, |
|
"pad_token": "<unk>", |
|
"padding_side": "right", |
|
"sp_model_kwargs": {}, |
|
"spaces_between_special_tokens": false, |
|
"tokenizer_class": "LlamaTokenizer", |
|
"unk_token": "<unk>", |
|
"use_default_system_prompt": false |
|
} |
|
|