Commit
·
927b82e
1
Parent(s):
bd6b737
Update README.md
Browse files
README.md
CHANGED
@@ -45,9 +45,8 @@ end_token = "<|"
|
|
45 |
tokenizer = GPT2Tokenizer.from_pretrained('gpt2-large')
|
46 |
model = GPT2LMHeadModel.from_pretrained('gpt2-large')
|
47 |
tokenizer.add_special_tokens({'pad_token': '[PAD]'})
|
48 |
-
tokenizer.add_special_tokens({'eos_token': '<|End|>'})
|
49 |
special_tokens = {
|
50 |
-
"additional_special_tokens": ["<|USER|>", "<|
|
51 |
}
|
52 |
tokenizer.add_special_tokens(special_tokens)
|
53 |
model.resize_token_embeddings(len(tokenizer))
|
|
|
45 |
tokenizer = GPT2Tokenizer.from_pretrained('gpt2-large')
|
46 |
model = GPT2LMHeadModel.from_pretrained('gpt2-large')
|
47 |
tokenizer.add_special_tokens({'pad_token': '[PAD]'})
|
|
|
48 |
special_tokens = {
|
49 |
+
"additional_special_tokens": ["<|USER|>", "<|ASSISTANT|>"]
|
50 |
}
|
51 |
tokenizer.add_special_tokens(special_tokens)
|
52 |
model.resize_token_embeddings(len(tokenizer))
|