Upload 2 files

QuestionAnswering.py

@@ -0,0 +1,493 @@
+from transformers import LukePreTrainedModel, LukeModel, AutoTokenizer, TrainingArguments, default_data_collator, Trainer, AutoModelForQuestionAnswering
+from transformers.modeling_outputs import ModelOutput
+from typing import Optional, Tuple, Union
+
+import numpy as np
+from tqdm import tqdm
+import evaluate
+import torch
+from dataclasses import dataclass
+from datasets import load_dataset, concatenate_datasets
+from torch import nn
+from torch.nn import CrossEntropyLoss
+import collections
+import re
+
+train = False
+test = True
+
+PEFT = False
+tf32 = True
+fp16 = True
+
+trained_model = "LUKE_squad_finetuned_qa_tf32"
+train_checkpoint = None
+squad_shift = False
+
+# For testing
+tokenizer_list = ["xlnet-base-cased"]
+model_list = ["botcon/XLNET_squad_finetuned_large"]
+question_list = ["who", "what", "where", "when", "which", "how", "whom", ".*"]
+
+base_tokenizer = "xlnet-base-cased"
+base_model = "studio-ousia/luke-base"
+
+# base_tokenizer = "xlnet-base-cased"
+# base_model = "xlnet-base-cased"
+
+# base_tokenizer = "bert-base-cased"
+# base_model = "SpanBERT/spanbert-base-cased"
+
+torch.backends.cuda.matmul.allow_tf32 = tf32
+torch.backends.cudnn.allow_tf32 = tf32
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+# https://github.com/huggingface/transformers/blob/v4.34.1/src/transformers/models/luke/modeling_luke.py#L319-L353
+# Taken from HF repository, easier to include additional features -- Currently identical to LukeForQuestionAnswering by H
+
+@dataclass
+class LukeQuestionAnsweringModelOutput(ModelOutput):
+    """
+    Outputs of question answering models.
+
+
+    Args:
+        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+            Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
+        start_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
+            Span-start scores (before SoftMax).
+        end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
+            Span-end scores (before SoftMax).
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+
+
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+            shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
+            layer plus the initial entity embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+    """
+
+
+    loss: Optional[torch.FloatTensor] = None
+    start_logits: torch.FloatTensor = None
+    end_logits: torch.FloatTensor = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    entity_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+
+class AugmentedLukeForQuestionAnswering(LukePreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        # This is 2.
+        self.num_labels = config.num_labels
+
+        self.luke = LukeModel(config, add_pooling_layer=False)
+
+        '''
+        Any improvement to the model are expected here. Additional features, anything...
+        '''
+        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
+        self.linear_dropout = nn.Dropout(0.1)
+
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.FloatTensor] = None,
+        entity_ids: Optional[torch.LongTensor] = None,
+        entity_attention_mask: Optional[torch.FloatTensor] = None,
+        entity_token_type_ids: Optional[torch.LongTensor] = None,
+        entity_position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        start_positions: Optional[torch.LongTensor] = None,
+        end_positions: Optional[torch.LongTensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, LukeQuestionAnsweringModelOutput]:
+        
+        r"""
+        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for position (index) of the start of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+            are not taken into account for computing the loss.
+        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for position (index) of the end of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+            are not taken into account for computing the loss.
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+
+        outputs = self.luke(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            entity_ids=entity_ids,
+            entity_attention_mask=entity_attention_mask,
+            entity_token_type_ids=entity_token_type_ids,
+            entity_position_ids=entity_position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=True,
+        )
+
+
+        sequence_output = outputs.last_hidden_state
+        sequence_output = self.linear_dropout(sequence_output)
+
+        logits = self.qa_outputs(sequence_output)
+        start_logits, end_logits = logits.split(1, dim=-1)
+        start_logits : torch.Tensor = start_logits.squeeze(-1)
+        end_logits = end_logits.squeeze(-1)
+
+
+        total_loss = None
+        if start_positions is not None and end_positions is not None:
+            # If we are on multi-GPU, split add a dimension
+            if len(start_positions.size()) > 1:
+                start_positions = start_positions.squeeze(-1)
+            if len(end_positions.size()) > 1:
+                end_positions = end_positions.squeeze(-1)
+            # sometimes the start/end positions are outside our model inputs, we ignore these terms
+            ignored_index = start_logits.size(1)
+            start_positions.clamp_(0, ignored_index)
+            end_positions.clamp_(0, ignored_index)
+
+            loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
+            start_loss = loss_fct(start_logits, start_positions)
+            end_loss = loss_fct(end_logits, end_positions)
+            total_loss = (start_loss + end_loss) / 2
+
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [
+                    total_loss,
+                    start_logits,
+                    end_logits,
+                    outputs.hidden_states,
+                    outputs.entity_hidden_states,
+                    outputs.attentions,
+                ]
+                if v is not None
+            )
+
+
+        return LukeQuestionAnsweringModelOutput(
+            loss=total_loss,
+            start_logits=start_logits,
+            end_logits=end_logits,
+            hidden_states=outputs.hidden_states,
+            entity_hidden_states=outputs.entity_hidden_states,
+            attentions=outputs.attentions,
+        )
+
+# Get data to train model - squadshift is designed as a validation/testing set, so there are multiple answers, take the shortest
+def get_squadshifts_training():
+    wiki = load_dataset("squadshifts", "new_wiki")["test"]
+    nyt = load_dataset("squadshifts", "nyt")["test"]
+    reddit = load_dataset("squadshifts", "reddit")["test"]
+    raw_dataset = concatenate_datasets([wiki, nyt, reddit])
+    updated = raw_dataset.map(validation_to_train)
+    return updated
+
+def validation_to_train(example):
+    answers = example["answers"]
+    answer_text = answers["text"]
+    index_min = min(range(len(answer_text)), key=lambda x : len(answer_text.__getitem__(x)))
+    answers["text"] = answers["text"][index_min:index_min+1]
+    answers["answer_start"] = answers["answer_start"][index_min:index_min+1]
+    return example
+
+# Get subset with specific question word
+def get_dataset(dataset, pattern):
+    return dataset.filter(lambda x : bool(re.search(r"\b{}\b".format(pattern), x["question"], flags=re.IGNORECASE)))
+
+if __name__ == "__main__":
+    # Setting up tokenizer and helper functions
+    # Work-around for FastTokenizer - RoBERTa and LUKE share the same subword vocab, and we are not using entities functions of LUKE-tokenizer anyways
+    tokenizer = AutoTokenizer.from_pretrained(base_tokenizer)
+
+    # Necessary initialization
+    max_length = 512
+    stride = 128
+    batch_size = 8
+    n_best = 20
+    max_answer_length = 30
+    metric = evaluate.load("squad")
+    raw_datasets = load_dataset("squad")
+
+    raw_train = raw_datasets["train"]
+    raw_validation = raw_datasets["validation"]
+
+    def compute_metrics(start_logits, end_logits, features, examples):
+        example_to_features = collections.defaultdict(list)
+        for idx, feature in enumerate(features):
+            example_to_features[feature["example_id"]].append(idx)
+
+        predicted_answers = []
+        for example in tqdm(examples):
+            example_id = example["id"]
+            context = example["context"]
+            answers = []
+
+            # Loop through all features associated with that example
+            for feature_index in example_to_features[example_id]:
+                start_logit = start_logits[feature_index]
+                end_logit = end_logits[feature_index]
+                offsets = features[feature_index]["offset_mapping"]
+
+                start_indexes = np.argsort(start_logit)[-1 : -n_best - 1 : -1].tolist()
+                end_indexes = np.argsort(end_logit)[-1 : -n_best - 1 : -1].tolist()
+                for start_index in start_indexes:
+                    for end_index in end_indexes:
+                        # Skip answers that are not fully in the context
+                        if offsets[start_index] is None or offsets[end_index] is None:
+                            continue
+                        # Skip answers with a length that is either < 0 or > max_answer_length
+                        if (
+                            end_index < start_index
+                            or end_index - start_index + 1 > max_answer_length
+                        ):
+                            continue
+
+                        answer = {
+                            "text": context[offsets[start_index][0] : offsets[end_index][1]],
+                            "logit_score": start_logit[start_index] + end_logit[end_index],
+                        }
+                        answers.append(answer)
+
+            # Select the answer with the best score
+            if len(answers) > 0:
+                best_answer = max(answers, key=lambda x: x["logit_score"])
+                predicted_answers.append(
+                    {"id": example_id, "prediction_text": best_answer["text"]}
+                )
+            else:
+                predicted_answers.append({"id": example_id, "prediction_text": ""})
+
+        theoretical_answers = [{"id": ex["id"], "answers": ex["answers"]} for ex in examples]
+        return metric.compute(predictions=predicted_answers, references=theoretical_answers)
+
+    def preprocess_training_examples(examples):
+    
+        questions = [q.strip() for q in examples["question"]]
+        inputs = tokenizer(
+            questions,
+            examples["context"],
+            max_length=max_length,
+            truncation="only_second",
+            stride=stride,
+            return_overflowing_tokens=True,
+            return_offsets_mapping=True,
+            padding="max_length",
+        )
+
+        offset_mapping = inputs.pop("offset_mapping")
+        sample_map = inputs.pop("overflow_to_sample_mapping")
+        answers = examples["answers"]
+        start_positions = []
+        end_positions = []
+
+        for i, offset in enumerate(offset_mapping):
+            sample_idx = sample_map[i]
+            answer = answers[sample_idx]
+            start_char = answer["answer_start"][0]
+            end_char = answer["answer_start"][0] + len(answer["text"][0])
+            sequence_ids = inputs.sequence_ids(i)
+
+            # Find the start and end of the context
+            idx = 0
+            while sequence_ids[idx] != 1:
+                idx += 1
+            context_start = idx
+            while sequence_ids[idx] == 1:
+                idx += 1
+            context_end = idx - 1
+
+            # If the answer is not fully inside the context, label is (0, 0)
+            if offset[context_start][0] > start_char or offset[context_end][1] < end_char:
+                start_positions.append(0)
+                end_positions.append(0)
+            else:
+                # Otherwise it's the start and end token positions
+                idx = context_start
+                while idx <= context_end and offset[idx][0] <= start_char:
+                    idx += 1
+                start_positions.append(idx - 1)
+
+                idx = context_end
+                while idx >= context_start and offset[idx][1] >= end_char:
+                    idx -= 1
+                end_positions.append(idx + 1)
+
+        inputs["start_positions"] = start_positions
+        inputs["end_positions"] = end_positions
+        return inputs
+
+    def preprocess_validation_examples(examples):
+        questions = [q.strip() for q in examples["question"]]
+        inputs = tokenizer(
+            questions,
+            examples["context"],
+            max_length=max_length,
+            truncation="only_second",
+            stride=stride,
+            return_overflowing_tokens=True,
+            return_offsets_mapping=True,
+            padding="max_length",
+        )
+
+
+        sample_map = inputs.pop("overflow_to_sample_mapping")
+        example_ids = []
+
+        for i in range(len(inputs["input_ids"])):
+            sample_idx = sample_map[i]
+            example_ids.append(examples["id"][sample_idx])
+
+            sequence_ids = inputs.sequence_ids(i)
+            offset = inputs["offset_mapping"][i]
+            inputs["offset_mapping"][i] = [
+                o if sequence_ids[k] == 1 else None for k, o in enumerate(offset)
+            ]
+
+        inputs["example_id"] = example_ids
+        return inputs
+
+    if train:
+
+        model = AutoModelForQuestionAnswering.from_pretrained(base_model).to(device)
+        model.train()
+
+        if squad_shift:
+            raw_train = get_squadshifts_training()
+
+        train_dataset = raw_train.map(
+            preprocess_training_examples,
+            batched=True,
+            remove_columns=raw_train.column_names,
+        )
+
+        validation_dataset = raw_validation.map(
+            preprocess_validation_examples,
+            batched=True,
+            remove_columns=raw_validation.column_names,
+        )
+
+        # --------------- PEFT -------------------- # One epoch without PEFT took about 2h on my computer with CUDA - performance of PEFT kinda ass though
+        if PEFT:
+            from peft import get_peft_config, get_peft_model, LoraConfig, TaskType
+
+            # ---- For all linear layers ----
+            import re
+            pattern = r'\((\w+)\): Linear'
+            linear_layers = re.findall(pattern, str(model.modules))
+            target_modules = list(set(linear_layers))
+
+            # If using peft, can consider increaisng r for better performance
+            peft_config = LoraConfig(
+                task_type=TaskType.QUESTION_ANS, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1, target_modules=target_modules, bias='all'
+            ) 
+
+            model = get_peft_model(model, peft_config)
+            model.print_trainable_parameters()
+
+            trained_model += "_PEFT"
+
+        # ------------------------------------------ #
+
+        args = TrainingArguments(
+            trained_model,
+            evaluation_strategy = "no",
+            save_strategy="epoch",
+            learning_rate=2e-5,
+            per_device_train_batch_size=batch_size,
+            per_device_eval_batch_size=batch_size,
+            num_train_epochs=3,
+            weight_decay=0.01,
+            push_to_hub=True,
+            fp16=fp16
+        )
+
+        trainer = Trainer(
+            model,
+            args,
+            train_dataset=train_dataset,
+            eval_dataset=validation_dataset,
+            data_collator=default_data_collator,
+            tokenizer=tokenizer
+        )
+
+        trainer.train(train_checkpoint)
+
+    if test:
+        out = "out.txt"
+        for j in range(1):
+            # model = AutoModelForQuestionAnswering.from_pretrained(model_list[j]).to(device)
+            # tokenizer = AutoTokenizer.from_pretrained(tokenizer_list[j])
+            # Normal case
+            # test_validation = raw_validation
+            for question in question_list:
+                model_name = "botcon/XLNET_squad_finetuned_large"
+                model = AutoModelForQuestionAnswering.from_pretrained(model_name).to(device)
+                model.eval()
+                tokenizer = AutoTokenizer.from_pretrained(base_tokenizer)
+                test_validation = get_dataset(raw_validation, question)
+                exact_match = 0
+                f1 = 0
+                validation_size = 50
+                start = 0
+                end = validation_size
+
+                with torch.no_grad():
+                    while start < len(test_validation):
+                        small_eval_set = test_validation.select(range(start, min(end, len(test_validation))))
+                        eval_set = small_eval_set.map(
+                            preprocess_validation_examples,
+                            batched=True,
+                            remove_columns=test_validation.column_names
+                        )
+                        eval_set_for_model = eval_set.remove_columns(["example_id", "offset_mapping"])
+                        eval_set_for_model.set_format("torch")
+                        batch = {k: eval_set_for_model[k].to(device) for k in eval_set_for_model.column_names}
+                        outputs = model(**batch)
+                        start_logits = outputs.start_logits.cpu().numpy()
+                        end_logits = outputs.end_logits.cpu().numpy()
+                        res = compute_metrics(start_logits, end_logits, eval_set, small_eval_set)
+                        exact_match += res['exact_match'] * (len(small_eval_set) / len(test_validation))
+                        f1 += res["f1"] * (len(small_eval_set) / len(test_validation))
+                        start += validation_size
+                        end += validation_size
+                    
+                    print("F1 score: {}".format(f1))
+                    print("Exact match: {}".format(exact_match))
+                with open(out, "a+") as file:
+                    file.write("Model: {},  Question: {}, Size: {}".format(model_name, question, len(test_validation)))
+                    file.write("\n")
+                    file.write("F1 score: {}".format(f1))
+                    file.write("\n")
+                    file.write("Exact match: {}".format(exact_match))
+                    file.write("\n")

README.md

@@ -0,0 +1,26 @@
+Main Script: QuestionAnswering.py
+
+The script uses HuggingFace library for managing the datasets, importing/exporting models and training the models.
+
+There are various variables at the start of the script.
+- train: Training a new model
+- PEFT: Whether to use PEFT during training
+- tf32/fp16: Mixed precision training choice
+- trained_model: Name of trained model (to be pushed to HF Hub)
+- train_checkpoint: Checkpoint of training (None by default)
+- squad_shift: Whether to include extra data (squadshift)
+- base_tokenizer: Tokenizer of base model
+- base_model: Pre-trained model
+
+- test: Testing a model
+- tokenizer_list/model_list/question_list: Which tokenizer, model and questions to be tested.
+
+CUDA is enabled if applicable.
+Require user to login into HuggingFace Hub (via command line token or through script) if training. Alternative is to not push to hub, a local repository will be created.
+
+Huggingface repositories created (models created)
+- botcon/XLNET_squad_finetuned_large
+- botcon/XLNET_squadshift_finetuned_large
+- botcon/LUKE_squad_finetuned_large
+- botcon/LUKE_squadshift_finetuned_large
+- botcon/LUKE_squad_what