alKoGolik's picture
Upload 169 files
c87c295 verified
raw
history blame contribute delete
314 Bytes
B_CODE = "[CODE_START_TOK]"
E_CODE = "[/CODE_END_TOK]"
B_RESULT = "[RESULT_TOK]"
E_RESULT = "[/RESULT_TOK]"
B_INST, E_INST = "[INST]", "[/INST]"
B_SYS, E_SYS = "<<SYS>>", "<</SYS>>"
IGNORE_INDEX = -100
DEFAULT_PAD_TOKEN = "[PAD]"
DEFAULT_EOS_TOKEN = "</s>"
DEFAULT_BOS_TOKEN = "<s>"
DEFAULT_UNK_TOKEN = "<unk>"