codefuse-ai

/

CodeFuse-13B

like 48

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("CodeFuse-13B")
Traceback (most recent call last):
File "", line 1, in
File "/data/anaconda3/envs/llama2/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 735, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
File "/data/anaconda3/envs/llama2/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 1854, in from_pretrained
return cls._from_pretrained(
File "/data/anaconda3/envs/llama2/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2017, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
File "/data/anaconda3/envs/llama2/lib/python3.10/site-packages/transformers/tokenization_utils_fast.py", line 111, in init
fast_tokenizer = TokenizerFast.from_file(fast_tokenizer_file)
Exception: expected value at line 1 column 1