Spaces:
Sleeping
Sleeping
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
""" | |
Sem-F1 metric | |
Author: Naman Bansal | |
""" | |
from typing import List, Optional, Tuple | |
import datasets | |
import evaluate | |
import nltk | |
import numpy as np | |
from numpy.typing import NDArray | |
from sklearn.metrics.pairwise import cosine_similarity | |
from .encoder_models import get_encoder | |
from .type_aliases import DEVICE_TYPE, PREDICTION_TYPE, REFERENCE_TYPE | |
from .utils import is_nested_list_of_type, Scores, slice_embeddings, flatten_list, get_gpu, sent_tokenize | |
_CITATION = """\ | |
@inproceedings{bansal-etal-2022-sem, | |
title = "{SEM}-F1: an Automatic Way for Semantic Evaluation of Multi-Narrative Overlap Summaries at Scale", | |
author = "Bansal, Naman and | |
Akter, Mousumi and | |
Karmaker Santu, Shubhra Kanti", | |
editor = "Goldberg, Yoav and | |
Kozareva, Zornitsa and | |
Zhang, Yue", | |
booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing", | |
month = dec, | |
year = "2022", | |
address = "Abu Dhabi, United Arab Emirates", | |
publisher = "Association for Computational Linguistics", | |
url = "https://aclanthology.org/2022.emnlp-main.49", | |
doi = "10.18653/v1/2022.emnlp-main.49", | |
pages = "780--792", | |
abstract = "Recent work has introduced an important yet relatively under-explored NLP task called Semantic Overlap Summarization (SOS) that entails generating a summary from multiple alternative narratives which conveys the common information provided by those narratives. Previous work also published a benchmark dataset for this task by collecting 2,925 alternative narrative pairs from the web and manually annotating 411 different reference summaries by engaging human annotators. In this paper, we exclusively focus on the automated evaluation of the SOS task using the benchmark dataset. More specifically, we first use the popular ROUGE metric from text-summarization literature and conduct a systematic study to evaluate the SOS task. Our experiments discover that ROUGE is not suitable for this novel task and therefore, we propose a new sentence-level precision-recall style automated evaluation metric, called SEM-F1 (Semantic F1). It is inspired by the benefits of the sentence-wise annotation technique using overlap labels reported by the previous work. Our experiments show that the proposed SEM-F1 metric yields a higher correlation with human judgment and higher inter-rater agreement compared to the ROUGE metric.", | |
} | |
""" | |
_DESCRIPTION = """\ | |
Sem-F1 metric leverages the pre-trained contextual embeddings and evaluates the model generated | |
semantic overlap summary with the reference overlap summary. It evaluates the semantic overlap summary at the | |
sentence level and computes precision, recall and F1 scores. | |
""" | |
_KWARGS_DESCRIPTION = """ | |
Sem-F1 compares the system-generated summaries (predictions) with ground truth reference summaries (references) | |
using precision, recall, and F1 score based on sentence embeddings. | |
Args: | |
predictions (list): List of predictions. Format varies based on `tokenize_sentences` and `multi_references` flags. | |
references (list): List of references. Format varies based on `tokenize_sentences` and `multi_references` flags. | |
model_type (str): Model to use for encoding sentences. Options: ['pv1', 'stsb', 'use'] | |
pv1 - paraphrase-distilroberta-base-v1 | |
stsb - stsb-roberta-large | |
use - Universal Sentence Encoder (Default) | |
Furthermore, you can use any model on Huggingface/SentenceTransformer that is supported by SentenceTransformer such | |
as `all-mpnet-base-v2` or `roberta-base` | |
tokenize_sentences (bool): Flag to indicate whether to tokenize the sentences in the input documents. Default: True. | |
multi_references (bool): Flag to indicate whether multiple references are provided. Default is False. | |
gpu (Union[bool, str, int, List[Union[str, int]]]): Whether to use GPU or CPU for computation. | |
bool - | |
False - CPU (Default) | |
True - GPU (device 0) if gpu is available else CPU | |
int - | |
n - GPU, device index n | |
str - | |
'cuda', 'gpu', 'cpu' | |
List[Union[str, int]] - Multiple GPUs/cpus i.e. use multiple processes when computing embeddings | |
batch_size (int): Batch size for encoding. Default is 32. | |
verbose (bool): Flag to indicate verbose output. Default is False. | |
Returns: | |
List of Scores dataclass with attributes as follows - | |
precision: float - precision score | |
recall: List[float] - List of recall scores corresponding to single/multiple references | |
f1: float - F1 score (between precision and average recall) | |
Examples of input formats: | |
Case 1: multi_references = False, tokenize_sentences = False | |
predictions: List[List[str]] - List of predictions where each prediction is a list of sentences. | |
references: List[List[str]] - List of references where each reference is a list of sentences. | |
Example: | |
predictions = [["This is a prediction sentence 1.", "This is a prediction sentence 2."]] | |
references = [["This is a reference sentence 1.", "This is a reference sentence 2."]] | |
Case 2: multi_references = False, tokenize_sentences = True | |
predictions: List[str] - List of predictions where each prediction is a document. | |
references: List[str] - List of references where each reference is a document. | |
Example: | |
predictions = ["This is a prediction sentence 1. This is a prediction sentence 2."] | |
references = ["This is a reference sentence 1. This is a reference sentence 2."] | |
Case 3: multi_references = True, tokenize_sentences = False | |
predictions: List[List[str]] - List of predictions where each prediction is a list of sentences. | |
references: List[List[List[str]]] - List of references where each example has multi-references (List[r1, r2, ...]) | |
and each ri is a List of sentences. | |
Example: | |
predictions = [["Prediction sentence 1.", "Prediction sentence 2."]] | |
references = [ | |
[ | |
["Reference sentence 1.", "Reference sentence 2."], # Reference 1 | |
["Alternative reference 1.", "Alternative reference 2."], # Reference 2 | |
] | |
] | |
Case 4: multi_references = True, tokenize_sentences = True | |
predictions: List[str] - List of predictions where each prediction is a document. | |
references: List[List[str]] - List of references where each example has multi-references (List[r1, r2, ...]) where | |
each r1 is a document. | |
Example: | |
predictions = ["Prediction sentence 1. Prediction sentence 2."] | |
references = [ | |
[ | |
"Reference sentence 1. Reference sentence 2.", # Reference 1 | |
"Alternative reference 1. Alternative reference 2.", # Reference 2 | |
] | |
] | |
Examples: | |
>>> import evaluate | |
>>> predictions = [ | |
["I go to School. You are stupid."], | |
["I love adventure sports."], | |
] | |
>>> references = [ | |
["I go to School. You are stupid."], | |
["I love outdoor sports."], | |
] | |
>>> metric = evaluate.load("nbansal/semf1") | |
>>> results = metric.compute(predictions=predictions, references=references) | |
>>> for score in results: | |
>>> print(f"Precision: {score.precision}, Recall: {score.recall}, F1: {score.f1}") | |
""" | |
def _compute_cosine_similarity(pred_embeds: NDArray, ref_embeds: NDArray) -> Tuple[float, float]: | |
""" | |
Compute precision and recall based on cosine similarity between predicted and reference embeddings. | |
Args: | |
pred_embeds (NDArray): Predicted embeddings (shape: [num_pred, embedding_dim]). | |
ref_embeds (NDArray): Reference embeddings (shape: [num_ref, embedding_dim]). | |
Returns: | |
Tuple[float, float]: Precision and recall based on cosine similarity scores. | |
Precision: Average maximum cosine similarity score per predicted embedding. | |
Recall: Average maximum cosine similarity score per reference embedding. | |
""" | |
# Compute cosine similarity between predicted and reference embeddings | |
cosine_scores = cosine_similarity(pred_embeds, ref_embeds) | |
# Compute precision per predicted embedding | |
precision_per_sentence_sim = np.max(cosine_scores, axis=-1) | |
# Compute recall per reference embedding | |
recall_per_sentence_sim = np.max(cosine_scores, axis=0) | |
# Calculate mean precision and recall scores | |
precision = np.mean(precision_per_sentence_sim).item() | |
recall = np.mean(recall_per_sentence_sim).item() | |
return precision, recall | |
def _validate_input_format( | |
tokenize_sentences: bool, | |
multi_references: bool, | |
predictions: PREDICTION_TYPE, | |
references: REFERENCE_TYPE, | |
): | |
""" | |
Validate the format of predictions and references based on specified criteria. | |
Args: | |
- tokenize_sentences (bool): Flag indicating whether sentences should be tokenized. | |
- multi_references (bool): Flag indicating whether multiple references are provided. | |
- predictions (PREDICTION_TYPE): Predictions to validate. | |
- references (REFERENCE_TYPE): References to validate. | |
Raises: | |
- ValueError: If the format of predictions or references does not meet the specified criteria. | |
Validation Criteria: | |
The function validates predictions and references based on the following conditions: | |
1. If `tokenize_sentences` is True and `multi_references` is True: | |
- Predictions must be a list of strings (`is_list_of_strings_at_depth(predictions, 1)`). | |
- References must be a list of list of strings (`is_list_of_strings_at_depth(references, 2)`). | |
2. If `tokenize_sentences` is False and `multi_references` is True: | |
- Predictions must be a list of list of strings (`is_list_of_strings_at_depth(predictions, 2)`). | |
- References must be a list of list of list of strings (`is_list_of_strings_at_depth(references, 3)`). | |
3. If `tokenize_sentences` is True and `multi_references` is False: | |
- Predictions must be a list of strings (`is_list_of_strings_at_depth(predictions, 1)`). | |
- References must be a list of strings (`is_list_of_strings_at_depth(references, 1)`). | |
4. If `tokenize_sentences` is False and `multi_references` is False: | |
- Predictions must be a list of list of strings (`is_list_of_strings_at_depth(predictions, 2)`). | |
- References must be a list of list of strings (`is_list_of_strings_at_depth(references, 2)`). | |
The function checks these conditions and raises a ValueError if any condition is not met, | |
indicating that predictions or references are not in the valid input format. | |
Note: | |
- `PREDICTION_TYPE` and `REFERENCE_TYPE` are defined at the top of the file | |
""" | |
if len(predictions) != len(references): | |
raise ValueError(f"Predictions and references must have the same length. " | |
f"Got {len(predictions)} predictions and {len(references)} references.") | |
if len(predictions) == 0: | |
raise ValueError("Can't have empty inputs") | |
def check_format(lst_obj, expected_depth: int, name: str): | |
is_valid, error_message = is_nested_list_of_type(lst_obj, element_type=str, depth=expected_depth) | |
if not is_valid: | |
raise ValueError(f"{name} are not in the expected format.\n" | |
f"Error: {error_message}.") | |
try: | |
if tokenize_sentences and multi_references: | |
check_format(predictions, 1, "Predictions") | |
check_format(references, 2, "References") | |
elif not tokenize_sentences and multi_references: | |
check_format(predictions, 2, "Predictions") | |
check_format(references, 3, "References") | |
elif tokenize_sentences and not multi_references: | |
check_format(predictions, 1, "Predictions") | |
check_format(references, 1, "References") | |
else: | |
check_format(predictions, 2, "Predictions") | |
check_format(references, 2, "References") | |
except ValueError as ve: | |
raise ValueError(f"Input validation error: {ve}") | |
class SemF1(evaluate.Metric): | |
_MODEL_TYPE_TO_NAME = { | |
"pv1": "paraphrase-distilroberta-base-v1", | |
"stsb": "stsb-roberta-large", | |
"use": "sentence-transformers/use-cmlm-multilingual", | |
} | |
def _info(self): | |
return evaluate.MetricInfo( | |
# This is the description that will appear on the modules page. | |
module_type="metric", | |
description=_DESCRIPTION, | |
citation=_CITATION, | |
inputs_description=_KWARGS_DESCRIPTION, | |
# This defines the format of each prediction and reference | |
features=[ | |
# F0: Multi References: False, Tokenize_Sentences = False | |
datasets.Features( | |
{ | |
# predictions: List[List[str]] - List of predictions where prediction is a list of sentences | |
"predictions": datasets.Sequence(datasets.Value("string", id="sequence"), id="predictions"), | |
# references: List[List[str]] - List of references where each reference is a list of sentences | |
"references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), | |
} | |
), | |
# F1: Multi References: False, Tokenize_Sentences = True | |
datasets.Features( | |
{ | |
# predictions: List[str] - List of predictions | |
"predictions": datasets.Value("string", id="sequence"), | |
# references: List[str] - List of documents | |
"references": datasets.Value("string", id="sequence"), | |
} | |
), | |
# F2: Multi References: True, Tokenize_Sentences = False | |
datasets.Features( | |
{ | |
# predictions: List[List[str]] - List of predictions where prediction is a list of sentences | |
"predictions": datasets.Sequence(datasets.Value("string", id="sequence"), id="predictions"), | |
# references: List[List[List[str]]] - List of multi-references. | |
# So each "reference" is also a list (r1, r2, ...). | |
# Further, each ri's are also list of sentences. | |
"references": datasets.Sequence( | |
datasets.Sequence(datasets.Value("string", id="sequence"), id="ref"), id="references"), | |
} | |
), | |
# F3: Multi References: True, Tokenize_Sentences = True | |
datasets.Features( | |
{ | |
# predictions: List[str] - List of predictions | |
"predictions": datasets.Value("string", id="sequence"), | |
# references: List[List[List[str]]] - List of multi-references. | |
# So each "reference" is also a list (r1, r2, ...). | |
"references": datasets.Sequence(datasets.Value("string", id="ref"), id="references"), | |
} | |
), | |
], | |
# # Homepage of the module for documentation | |
# Additional links to the codebase or references | |
reference_urls=["https://aclanthology.org/2022.emnlp-main.49/"] | |
) | |
def _get_model_name(self, model_type: Optional[str] = None) -> str: | |
if model_type is None: | |
model_type = "use" | |
if model_type not in self._MODEL_TYPE_TO_NAME.keys(): | |
return model_type | |
return self._MODEL_TYPE_TO_NAME[model_type] | |
def _download_and_prepare(self, dl_manager): | |
"""Optional: download external resources useful to compute the scores""" | |
import nltk | |
nltk.download("punkt", quiet=True) | |
def _compute( | |
self, | |
predictions, | |
references, | |
model_type: Optional[str] = None, | |
tokenize_sentences: bool = True, | |
multi_references: bool = False, | |
gpu: DEVICE_TYPE = False, | |
batch_size: int = 32, | |
verbose: bool = False, | |
aggregate: bool = False, | |
) -> List[Scores]: | |
""" | |
Compute precision, recall, and F1 scores for given predictions and references. | |
:param predictions | |
:param references | |
:param model_type: Type of model to use for encoding. | |
Options: [pv1, stsb, use] | |
pv1 - paraphrase-distilroberta-base-v1 | |
stsb - stsb-roberta-large | |
use - Universal Sentence Encoder (Default) | |
Furthermore, you can use any model on Huggingface/SentenceTransformer that is supported by | |
SentenceTransformer. | |
:param tokenize_sentences: Flag to sentence tokenize the document. | |
:param multi_references: Flag to indicate multiple references. | |
:param gpu: GPU device to use. | |
:param batch_size: Batch size for encoding. | |
:param verbose: Flag to indicate verbose output. | |
:param aggregate: Flag to determine if output should be averaged | |
:return: List of Scores dataclass with precision, recall, and F1 scores. | |
""" | |
# Note: I have to specifically handle this case because the library considers the feature corresponding to | |
# this case (F2) as the feature for the other case (F0) i.e. it can't make any distinction between | |
# List[str] and List[List[str]] | |
if not tokenize_sentences and multi_references: | |
references = [[eval(ref) for ref in mul_ref_ex] for mul_ref_ex in references] | |
# Validate inputs corresponding to flags | |
_validate_input_format(tokenize_sentences, multi_references, predictions, references) | |
# Get GPU | |
device = get_gpu(gpu) | |
if verbose: | |
print(f"Using devices: {device}") | |
# Get the encoder model | |
model_name = self._get_model_name(model_type) | |
encoder = get_encoder(model_name, device=device, batch_size=batch_size, verbose=verbose) | |
# We'll handle the single reference and multi-reference case same way. So change the data format accordingly | |
if not multi_references: | |
references = [[ref] for ref in references] | |
# Tokenize sentences if required | |
if tokenize_sentences: | |
predictions = [sent_tokenize(pred) for pred in predictions] | |
references = [[sent_tokenize(ref) for ref in refs] for refs in references] | |
# Flatten the data for batch processing | |
all_sentences = flatten_list(predictions) + flatten_list(references) | |
# Get num of sentences to get the corresponding embeddings | |
prediction_sentences_count = [len(pred) for pred in predictions] | |
reference_sentences_count = [[len(ref) for ref in refs] for refs in references] | |
# Note: This is the most optimal way of doing it | |
# Encode all sentences in one go | |
embeddings = encoder.encode(all_sentences) | |
# Get embeddings corresponding to predictions and references | |
pred_embeddings = slice_embeddings(embeddings, prediction_sentences_count) | |
ref_embeddings = slice_embeddings(embeddings[sum(prediction_sentences_count):], reference_sentences_count) | |
# Init output scores | |
results = [] | |
# Compute scores | |
for preds, refs in zip(pred_embeddings, ref_embeddings): | |
# Precision: Concatenate all the sentences in all the references | |
concat_refs = np.concatenate(refs, axis=0) | |
precision, _ = _compute_cosine_similarity(preds, concat_refs) | |
precision = np.clip(precision, a_min=0.0, a_max=1.0).item() | |
# Recall: Compute individually for each reference | |
recall_scores = [_compute_cosine_similarity(r_embeds, preds) for r_embeds in refs] | |
recall_scores = [np.clip(r_scores, 0.0, 1.0).item() for (r_scores, _) in recall_scores] | |
results.append(Scores(precision, recall_scores)) | |
# runn aggregation procedure | |
if aggregate: | |
mean_prec = np.mean( | |
[score.precision for score in results] | |
) | |
mean_recall = np.mean(np.concatenate( | |
[np.array(score.recall) for score in results] | |
)) | |
aggregated_score = Scores( | |
float(mean_prec), | |
[float(mean_recall)] | |
) | |
aggregated_score.f1 = float(np.mean( | |
[score.f1 for score in results] | |
)) | |
results = aggregated_score | |
return results |