Update README.md

README.md

@@ -30,7 +30,9 @@ from transformers import AutoModel, AutoTokenizer
 checkpoint = "codesage/codesage-base"
 device = "cuda"  # for GPU usage or "cpu" for CPU usage
 
-tokenizer = AutoTokenizer.from_pretrained(checkpoint, trust_remote_code=True)
+# CodeSage requires adding eos token at the end of each tokenized sequence to ensure good performance
+tokenizer = AutoTokenizer.from_pretrained(checkpoint, trust_remote_code=True, add_eos_token=True)
+
 model = AutoModel.from_pretrained(checkpoint, trust_remote_code=True).to(device)
 
 inputs = tokenizer.encode("def print_hello_world():\tprint('Hello World!')", return_tensors="pt").to(device)