Locutusque commited on
Commit
927b82e
·
1 Parent(s): bd6b737

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +1 -2
README.md CHANGED
@@ -45,9 +45,8 @@ end_token = "<|"
45
  tokenizer = GPT2Tokenizer.from_pretrained('gpt2-large')
46
  model = GPT2LMHeadModel.from_pretrained('gpt2-large')
47
  tokenizer.add_special_tokens({'pad_token': '[PAD]'})
48
- tokenizer.add_special_tokens({'eos_token': '<|End|>'})
49
  special_tokens = {
50
- "additional_special_tokens": ["<|USER|>", "<|SYSTEM|>", "<|ASSISTANT|>"]
51
  }
52
  tokenizer.add_special_tokens(special_tokens)
53
  model.resize_token_embeddings(len(tokenizer))
 
45
  tokenizer = GPT2Tokenizer.from_pretrained('gpt2-large')
46
  model = GPT2LMHeadModel.from_pretrained('gpt2-large')
47
  tokenizer.add_special_tokens({'pad_token': '[PAD]'})
 
48
  special_tokens = {
49
+ "additional_special_tokens": ["<|USER|>", "<|ASSISTANT|>"]
50
  }
51
  tokenizer.add_special_tokens(special_tokens)
52
  model.resize_token_embeddings(len(tokenizer))