Tokenization updates

Some more suggestions for updates

tokenization_xgen.py

@@ -115,7 +115,6 @@ class XgenTokenizer(PreTrainedTokenizer):
     def __init__(
             self,
             pad_token=None,
-            eos_token="<|endoftext|>",
             add_eos_token=False,
             add_special_tokens=True,
             **kwargs,
@@ -149,20 +148,22 @@ class XgenTokenizer(PreTrainedTokenizer):
     def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""
         if isinstance(token, str):
-            ids = self._tokenize(token)
-            return ids[0]
-        return token
+            return self.encoder.encode_single_token(token)
+        else:
+            return token
 
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
         return self.encoder.decode_single_token_bytes(index)
 
     def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
+        if skip_special_tokens:
+            token_ids = [t for t in token_ids if t not in self.all_special_ids]
         return self.encoder.decode(token_ids)
 
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
         """Build model inputs from a sequence by appending eos_token_id."""
-        eos_token_id = [50256] if self.add_eos_token else []
+        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
 
         output = token_ids_0 + eos_token_id
 
@@ -218,7 +219,7 @@ class XgenTokenizer(PreTrainedTokenizer):
         Returns:
             `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
         """
-        eos_token_id = [50256] if self.add_eos_token else []
+        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
 
         output = [0] * len(token_ids_0 + eos_token_id)