Spaces:
No application file
No application file
import tqdm | |
import logging | |
from dataclasses import dataclass | |
from typing import Optional, List, Any, Union | |
from transformers import PreTrainedTokenizer | |
logger = logging.getLogger(__name__) | |
class TextExample: | |
""" | |
A single training/test example for simple sequence classification. | |
Args: | |
guid: Unique id for the example. | |
text_a: string. The untokenized text of the first sequence. For single | |
sequence tasks, only this sequence must be specified. | |
text_b: (Optional) string. The untokenized text of the second sequence. | |
Only must be specified for sequence pair tasks. | |
label: (Optional) string. The label of the example. This should be | |
specified for train and dev examples, but not for test examples. | |
""" | |
guid: str | |
text_a: str | |
text_b: Optional[str] = None | |
label: Optional[str] = None | |
def to_json_string(self): | |
"""Serializes this instance to a JSON string.""" | |
return json.dumps(dataclasses.asdict(self), indent=2) + "\n" | |
class MultipleChoiceExample: | |
""" | |
A single training/test example for multiple choice | |
Args: | |
example_id: Unique id for the example. | |
question: string. The untokenized text of the second sequence | |
(question). | |
contexts: list of str. The untokenized text of the first sequence | |
(context of corresponding question). | |
endings: list of str. multiple choice's options. Its length must be | |
equal to contexts' length. | |
label: (Optional) string. The label of the example. This should be | |
specified for train and dev examples, but not for test examples. | |
""" | |
example_id: str | |
question: str | |
contexts: List[str] | |
endings: List[str] | |
label: Optional[str] | |
class TokensExample: | |
""" | |
A single training/test example for token classification. | |
Args: | |
guid: Unique id for the example. | |
words: list. The words of the sequence. | |
labels: (Optional) list. The labels for each word of the sequence. This | |
should be specified for train and dev examples, but not for test | |
examples. | |
""" | |
guid: str | |
words: List[str] | |
labels: Optional[List[str]] | |
class InputFeatures: | |
""" | |
A single set of features of data. | |
Property names are the same names as the corresponding inputs to a model. | |
""" | |
input_ids: Any | |
attention_mask: Any | |
token_type_ids: Any = None | |
label: Any = None | |
candidates: Any = None | |
example_id: str = None | |
def convert_multiple_choice_examples_to_features( | |
examples: List[MultipleChoiceExample], | |
tokenizer: PreTrainedTokenizer, | |
max_length: int, | |
label_list: List[str], | |
pad_token_segment_id=0, | |
pad_on_left=False, | |
pad_token=0, | |
mask_padding_with_zero=True, | |
) -> List[InputFeatures]: | |
""" | |
Loads a data file into a list of `InputFeatures` | |
""" | |
label_map = {label: i for i, label in enumerate(label_list)} | |
features = [] | |
for (ex_index, example) in tqdm.tqdm(enumerate(examples), desc="convert examples to features"): | |
if ex_index % 10000 == 0: | |
logger.info("Writing example %d of %d" % (ex_index, len(examples))) | |
choices_inputs = [] | |
for ending_idx, (context, ending) in enumerate(zip(example.contexts, example.endings)): | |
text_a = context | |
if example.question.find("_") != -1: | |
# this is for cloze question | |
text_b = example.question.replace("_", ending) | |
else: | |
text_b = example.question + " " + ending | |
inputs = tokenizer( | |
text_a, | |
text_b, | |
add_special_tokens=True, | |
max_length=max_length, | |
truncation='longest_first', | |
pad_to_max_length=True, | |
) | |
if "num_truncated_tokens" in inputs and inputs["num_truncated_tokens"] > 0: | |
logger.info( | |
"Attention! you are cropping tokens (swag task is ok). " | |
"If you are training ARC and RACE and you are poping question + options," | |
"you need to try to use a bigger max seq length!" | |
) | |
choices_inputs.append(inputs) | |
label = label_map[example.label] | |
input_ids = [x["input_ids"] for x in choices_inputs] | |
attention_mask = ( | |
[x["attention_mask"] for x in choices_inputs] if "attention_mask" in choices_inputs[0] else None | |
) | |
token_type_ids = ( | |
[x["token_type_ids"] for x in choices_inputs] if "token_type_ids" in choices_inputs[0] else None | |
) | |
features.append( | |
InputFeatures( | |
example_id=example.example_id, | |
input_ids=input_ids, | |
attention_mask=attention_mask, | |
token_type_ids=token_type_ids, | |
label=label, | |
) | |
) | |
for f in features[:2]: | |
logger.info("*** Example ***") | |
logger.info("feature: %s" % f) | |
return features | |
def convert_tokens_examples_to_features( | |
examples: List[TokensExample], | |
label_list: List[str], | |
max_seq_length: int, | |
tokenizer: PreTrainedTokenizer, | |
cls_token_at_end=False, | |
cls_token='[CLS]', | |
cls_token_segment_id=1, | |
sep_token='[SEP]', | |
sep_token_extra=False, | |
pad_on_left=False, | |
pad_token=0, | |
pad_token_segment_id=0, | |
pad_token_label_id=-100, | |
sequence_a_segment_id=0, | |
mask_padding_with_zero=True, | |
) -> List[InputFeatures]: | |
""" Loads a data file into a list of `InputFeatures` | |
`cls_token_at_end` define the location of the CLS token: | |
- False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP] | |
- True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS] | |
`cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet) | |
""" | |
# TODO clean up all this to leverage built-in features of tokenizers | |
label_map = {label: i for i, label in enumerate(label_list)} | |
features = [] | |
for (ex_index, example) in enumerate(examples): | |
if ex_index % 10_000 == 0: | |
logger.info("Writing example %d of %d", ex_index, len(examples)) | |
tokens = [] | |
label_ids = [] | |
for word, label in zip(example.words, example.labels): | |
word_tokens = tokenizer.tokenize(word) | |
# bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space. | |
if len(word_tokens) > 0: | |
tokens.extend(word_tokens) | |
# Use the real label id for the first token of the word, and padding ids for the remaining tokens | |
label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(word_tokens) - 1)) | |
# Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa. | |
special_tokens_count = tokenizer.num_special_tokens_to_add() | |
if len(tokens) > max_seq_length - special_tokens_count: | |
tokens = tokens[: (max_seq_length - special_tokens_count)] | |
label_ids = label_ids[: (max_seq_length - special_tokens_count)] | |
# The convention in BERT is: | |
# (a) For sequence pairs: | |
# tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] | |
# type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 | |
# (b) For single sequences: | |
# tokens: [CLS] the dog is hairy . [SEP] | |
# type_ids: 0 0 0 0 0 0 0 | |
# | |
# Where "type_ids" are used to indicate whether this is the first | |
# sequence or the second sequence. The embedding vectors for `type=0` and | |
# `type=1` were learned during pre-training and are added to the wordpiece | |
# embedding vector (and position vector). This is not *strictly* necessary | |
# since the [SEP] token unambiguously separates the sequences, but it makes | |
# it easier for the model to learn the concept of sequences. | |
# | |
# For classification tasks, the first vector (corresponding to [CLS]) is | |
# used as as the "sentence vector". Note that this only makes sense because | |
# the entire model is fine-tuned. | |
tokens += [sep_token] | |
label_ids += [pad_token_label_id] | |
if sep_token_extra: | |
# roberta uses an extra separator b/w pairs of sentences | |
tokens += [sep_token] | |
label_ids += [pad_token_label_id] | |
segment_ids = [sequence_a_segment_id] * len(tokens) | |
if cls_token_at_end: | |
tokens += [cls_token] | |
label_ids += [pad_token_label_id] | |
segment_ids += [cls_token_segment_id] | |
else: | |
tokens = [cls_token] + tokens | |
label_ids = [pad_token_label_id] + label_ids | |
segment_ids = [cls_token_segment_id] + segment_ids | |
input_ids = tokenizer.convert_tokens_to_ids(tokens) | |
# The mask has 1 for real tokens and 0 for padding tokens. Only real | |
# tokens are attended to. | |
input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) | |
# Zero-pad up to the sequence length. | |
padding_length = max_seq_length - len(input_ids) | |
if pad_on_left: | |
input_ids = ([pad_token] * padding_length) + input_ids | |
input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask | |
segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids | |
label_ids = ([pad_token_label_id] * padding_length) + label_ids | |
else: | |
input_ids += [pad_token] * padding_length | |
input_mask += [0 if mask_padding_with_zero else 1] * padding_length | |
segment_ids += [pad_token_segment_id] * padding_length | |
label_ids += [pad_token_label_id] * padding_length | |
assert len(input_ids) == max_seq_length | |
assert len(input_mask) == max_seq_length | |
assert len(segment_ids) == max_seq_length | |
assert len(label_ids) == max_seq_length | |
if ex_index < 5: | |
logger.info("*** Example ***") | |
logger.info("guid: %s", example.guid) | |
logger.info("tokens: %s", " ".join([str(x) for x in tokens])) | |
logger.info("input_ids: %s", " ".join([str(x) for x in input_ids])) | |
logger.info("input_mask: %s", " ".join([str(x) for x in input_mask])) | |
logger.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) | |
logger.info("label_ids: %s", " ".join([str(x) for x in label_ids])) | |
if "token_type_ids" not in tokenizer.model_input_names: | |
segment_ids = None | |
features.append( | |
InputFeatures( | |
input_ids=input_ids, attention_mask=input_mask, token_type_ids=segment_ids, label=label_ids | |
) | |
) | |
return features | |
def convert_text_examples_to_features( | |
examples: List[TextExample], | |
tokenizer: PreTrainedTokenizer, | |
max_length: Optional[int] = None, | |
label_list=None, | |
output_mode=None, | |
): | |
if max_length is None: | |
max_length = tokenizer.model_max_length | |
label_map = {label: i for i, label in enumerate(label_list)} | |
def label_from_example(example: TextExample) -> Union[int, float, None]: | |
if example.label is None: | |
return None | |
if output_mode == "classification": | |
return label_map[example.label] | |
elif output_mode == "regression": | |
return float(example.label) | |
raise KeyError(output_mode) | |
labels = [label_from_example(example) for example in examples] | |
batch_encoding = tokenizer( | |
[example.text_a if example.text_b is None else (example.text_a, example.text_b) for example in examples], | |
max_length=max_length, | |
padding="max_length", | |
truncation=True, | |
) | |
features = [] | |
for i in range(len(examples)): | |
inputs = {k: batch_encoding[k][i] for k in batch_encoding} | |
feature = InputFeatures(**inputs, label=labels[i]) | |
features.append(feature) | |
return features |