tokenizer_config.json

{"do_lower_case": true, "bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "unk_token": {"content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "tokenize_chinese_chars": true, "strip_accents": null, "clean_up_tokenization_spaces": true, "model_max_length": 512, "tokenizer_class": "MPNetTokenizer", "special_tokens_map_file": "output/is2b_medcycle1_sentence-transformers-all-mpnet-base-v2-2023-09-26_00-14-04/special_tokens_map.json", "name_or_path": "output/biosses1pearson_medsts_sick_sts_mednlisim_medsts_is2b_medcycle1_sentence-transformers-all-mpnet-base-v2-2023-09-26_09-20-25/0_Transformer/", "do_basic_tokenize": true, "never_split": null}