| from transformers import AutoTokenizer | |
| from vocab import TokenizerType | |
| tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan-7B", trust_remote_code=True) | |
| # byte-bpe sentencepiece | |
| tokenizer.type = TokenizerType.ByteBPE | |
| tokenizer.comments = "使用 SentencePiece 中的 Byte-Pair Encoding (BPE) 作为分词算法" | |