RANITBAG commited on
Commit
d414ee2
1 Parent(s): da5e853

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.py +246 -0
  2. tokenizer_config.json +6 -0
tokenizer.py ADDED
@@ -0,0 +1,246 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from shutil import copyfile
3
+ from typing import Any, Dict, List, Optional, Tuple
4
+
5
+ import sentencepiece as spm
6
+ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
7
+ from transformers.utils import logging
8
+
9
+ logger = logging.get_logger(__name__)
10
+
11
+ VOCAB_FILES_NAMES = {"vocab_file": "Tokenizer.model"}
12
+
13
+ PRETRAINED_VOCAB_FILES_MAP = {
14
+ "vocab_file": {},
15
+ "tokenizer_file": {},
16
+ }
17
+ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
18
+
19
+
20
+ class CustomAITokenizer(PreTrainedTokenizer):
21
+ """
22
+ Construct a custom tokenizer. Based on byte-level Byte-Pair-Encoding.
23
+ Args:
24
+ vocab_file (`str`):
25
+ Path to the vocabulary file.
26
+ """
27
+
28
+ vocab_files_names = VOCAB_FILES_NAMES
29
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
30
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
31
+ model_input_names = ["input_ids", "attention_mask"]
32
+
33
+ def __init__(
34
+ self,
35
+ vocab_file,
36
+ unk_token="<unk>",
37
+ bos_token="<|startoftext|>",
38
+ eos_token="<|endoftext|>",
39
+ pad_token="<unk>",
40
+ sp_model_kwargs: Optional[Dict[str, Any]] = None,
41
+ add_bos_token=True,
42
+ add_eos_token=False,
43
+ clean_up_tokenization_spaces=False,
44
+ **kwargs,
45
+ ):
46
+ self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
47
+ bos_token = (
48
+ AddedToken(bos_token, lstrip=False, rstrip=False)
49
+ if isinstance(bos_token, str)
50
+ else bos_token
51
+ )
52
+ eos_token = (
53
+ AddedToken(eos_token, lstrip=False, rstrip=False)
54
+ if isinstance(eos_token, str)
55
+ else eos_token
56
+ )
57
+ unk_token = (
58
+ AddedToken(unk_token, lstrip=False, rstrip=False)
59
+ if isinstance(unk_token, str)
60
+ else unk_token
61
+ )
62
+ pad_token = (
63
+ AddedToken(pad_token, lstrip=False, rstrip=False)
64
+ if isinstance(pad_token, str)
65
+ else pad_token
66
+ )
67
+ self.vocab_file = vocab_file
68
+ self.add_bos_token = add_bos_token
69
+ self.add_eos_token = add_eos_token
70
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
71
+ self.sp_model.Load(vocab_file)
72
+ super().__init__(
73
+ bos_token=bos_token,
74
+ eos_token=eos_token,
75
+ unk_token=unk_token,
76
+ pad_token=pad_token,
77
+ add_bos_token=add_bos_token,
78
+ add_eos_token=add_eos_token,
79
+ sp_model_kwargs=self.sp_model_kwargs,
80
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
81
+ **kwargs,
82
+ )
83
+
84
+ def __getstate__(self):
85
+ state = self.__dict__.copy()
86
+ state["sp_model"] = None
87
+ return state
88
+
89
+ def __setstate__(self, d):
90
+ self.__dict__ = d
91
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
92
+ self.sp_model.Load(self.vocab_file)
93
+
94
+ @property
95
+ def vocab_size(self):
96
+ """Returns vocab size"""
97
+ return self.sp_model.get_piece_size()
98
+
99
+ def get_vocab(self):
100
+ """Returns vocab as a dict"""
101
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
102
+ vocab.update(self.added_tokens_encoder)
103
+ return vocab
104
+
105
+ def _tokenize(self, text):
106
+ """Returns a tokenized string."""
107
+ return self.sp_model.encode(text, out_type=str)
108
+
109
+ def _convert_token_to_id(self, token):
110
+ """Converts a token (str) in an id using the vocab."""
111
+ return self.sp_model.piece_to_id(token)
112
+
113
+ def _convert_id_to_token(self, index):
114
+ """Converts an index (integer) in a token (str) using the vocab."""
115
+ token = self.sp_model.IdToPiece(index)
116
+ return token
117
+
118
+ def convert_tokens_to_string(self, tokens):
119
+ """Converts a sequence of tokens (string) in a single string."""
120
+ current_sub_tokens = []
121
+ out_string = ""
122
+ prev_is_special = False
123
+ for i, token in enumerate(tokens):
124
+ # make sure that special tokens are not decoded using sentencepiece model
125
+ if token in self.all_special_tokens:
126
+ if not prev_is_special and i != 0:
127
+ out_string += " "
128
+ out_string += self.sp_model.decode(current_sub_tokens) + token
129
+ prev_is_special = True
130
+ current_sub_tokens = []
131
+ else:
132
+ current_sub_tokens.append(token)
133
+ prev_is_special = False
134
+ out_string += self.sp_model.decode(current_sub_tokens)
135
+ return out_string
136
+
137
+ def save_vocabulary(
138
+ self, save_directory, filename_prefix: Optional[str] = None
139
+ ) -> Tuple[str]:
140
+ """
141
+ Save the vocabulary and special tokens file to a directory.
142
+ Args:
143
+ save_directory (`str`):
144
+ The directory in which to save the vocabulary.
145
+ Returns:
146
+ `Tuple(str)`: Paths to the files saved.
147
+ """
148
+ if not os.path.isdir(save_directory):
149
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
150
+ return
151
+ out_vocab_file = os.path.join(
152
+ save_directory,
153
+ (filename_prefix + "-" if filename_prefix else "")
154
+ + VOCAB_FILES_NAMES["vocab_file"],
155
+ )
156
+
157
+ if os.path.abspath(self.vocab_file) != os.path.abspath(
158
+ out_vocab_file
159
+ ) and os.path.isfile(self.vocab_file):
160
+ copyfile(self.vocab_file, out_vocab_file)
161
+ elif not os.path.isfile(self.vocab_file):
162
+ with open(out_vocab_file, "wb") as fi:
163
+ content_spiece_model = self.sp_model.serialized_model_proto()
164
+ fi.write(content_spiece_model)
165
+
166
+ return (out_vocab_file,)
167
+
168
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
169
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
170
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
171
+
172
+ output = bos_token_id + token_ids_0 + eos_token_id
173
+
174
+ if token_ids_1 is not None:
175
+ output = output + bos_token_id + token_ids_1 + eos_token_id
176
+
177
+ return output
178
+
179
+ def get_special_tokens_mask(
180
+ self,
181
+ token_ids_0: List[int],
182
+ token_ids_1: Optional[List[int]] = None,
183
+ already_has_special_tokens: bool = False,
184
+ ) -> List[int]:
185
+ """
186
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
187
+ special tokens using the tokenizer `prepare_for_model` method.
188
+ Args:
189
+ token_ids_0 (`List[int]`):
190
+ List of IDs.
191
+ token_ids_1 (`List[int]`, *optional*):
192
+ Optional second list of IDs for sequence pairs.
193
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
194
+ Whether or not the token list is already formatted with special tokens for the model.
195
+ Returns:
196
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
197
+ """
198
+ if already_has_special_tokens:
199
+ return super().get_special_tokens_mask(
200
+ token_ids_0=token_ids_0,
201
+ token_ids_1=token_ids_1,
202
+ already_has_special_tokens=True,
203
+ )
204
+
205
+ bos_token_id = [1] if self.add_bos_token else []
206
+ eos_token_id = [1] if self.add_eos_token else []
207
+
208
+ if token_ids_1 is None:
209
+ return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
210
+ return (
211
+ bos_token_id
212
+ + ([0] * len(token_ids_0))
213
+ + eos_token_id
214
+ + bos_token_id
215
+ + ([0] * len(token_ids_1))
216
+ + eos_token_id
217
+ )
218
+
219
+ def create_token_type_ids_from_sequences(
220
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
221
+ ) -> List[int]:
222
+ """
223
+ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
224
+ sequence pair mask has the following format:
225
+ ```
226
+ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
227
+ | first sequence | second sequence |
228
+ ```
229
+ if token_ids_1 is None, only returns the first portion of the mask (0s).
230
+ Args:
231
+ token_ids_0 (`List[int]`):
232
+ List of ids.
233
+ token_ids_1 (`List[int]`, *optional*):
234
+ Optional second list of IDs for sequence pairs.
235
+ Returns:
236
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
237
+ """
238
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
239
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
240
+
241
+ output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
242
+
243
+ if token_ids_1 is not None:
244
+ output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
245
+
246
+ return output
tokenizer_config.json CHANGED
@@ -27,6 +27,12 @@
27
  "special": true
28
  }
29
  },
 
 
 
 
 
 
30
  "bos_token": "<|startoftext|>",
31
  "clean_up_tokenization_spaces": false,
32
  "eos_token": "<|endoftext|>",
 
27
  "special": true
28
  }
29
  },
30
+ "auto_map": {
31
+ "AutoTokenizer": [
32
+ "tokenizer.CustomAITokenizer",
33
+ null
34
+ ]
35
+ },
36
  "bos_token": "<|startoftext|>",
37
  "clean_up_tokenization_spaces": false,
38
  "eos_token": "<|endoftext|>",