Spaces:
Running
Running
| # Import necessary libraries | |
| import nltk | |
| import numpy as np | |
| import torch | |
| import matplotlib.pyplot as plt | |
| from scipy.special import rel_entr | |
| from collections import Counter | |
| from transformers import GPT2LMHeadModel, GPT2TokenizerFast | |
| distortion_val={} | |
| # Download NLTK data if not already present | |
| nltk.download('punkt', quiet=True) | |
| class SentenceDistortionCalculator: | |
| """ | |
| A class to calculate and analyze distortion metrics between an original sentence and modified sentences. | |
| """ | |
| def __init__(self, original_sentence, modified_sentences): | |
| """ | |
| Initialize the calculator with the original sentence and a list of modified sentences. | |
| """ | |
| self.original_sentence = original_sentence | |
| self.modified_sentences = modified_sentences | |
| # Raw metric dictionaries | |
| self.levenshtein_distances = {} | |
| self.word_level_changes = {} | |
| self.kl_divergences = {} | |
| self.perplexities = {} | |
| # Normalized metric dictionaries | |
| self.normalized_levenshtein = {} | |
| self.normalized_word_changes = {} | |
| self.normalized_kl_divergences = {} | |
| self.normalized_perplexities = {} | |
| # Combined distortion dictionary | |
| self.combined_distortions = {} | |
| # Initialize GPT-2 model and tokenizer for perplexity calculation | |
| self.tokenizer = GPT2TokenizerFast.from_pretrained("gpt2") | |
| self.model = GPT2LMHeadModel.from_pretrained("gpt2") | |
| self.model.eval() # Set model to evaluation mode | |
| def calculate_all_metrics(self): | |
| """ | |
| Calculate all distortion metrics for each modified sentence. | |
| """ | |
| for idx, modified_sentence in enumerate(self.modified_sentences): | |
| key = f"Sentence_{idx+1}" | |
| self.levenshtein_distances[key] = self._calculate_levenshtein_distance(modified_sentence) | |
| self.word_level_changes[key] = self._calculate_word_level_change(modified_sentence) | |
| self.kl_divergences[key] = self._calculate_kl_divergence(modified_sentence) | |
| self.perplexities[key] = self._calculate_perplexity(modified_sentence) | |
| def normalize_metrics(self): | |
| """ | |
| Normalize all metrics to be between 0 and 1. | |
| """ | |
| self.normalized_levenshtein = self._normalize_dict(self.levenshtein_distances) | |
| self.normalized_word_changes = self._normalize_dict(self.word_level_changes) | |
| self.normalized_kl_divergences = self._normalize_dict(self.kl_divergences) | |
| self.normalized_perplexities = self._normalize_dict(self.perplexities) | |
| def calculate_combined_distortion(self): | |
| """ | |
| Calculate the combined distortion using the root mean square of the normalized metrics. | |
| """ | |
| for key in self.normalized_levenshtein.keys(): | |
| rms = np.sqrt( | |
| ( | |
| self.normalized_levenshtein[key] ** 2 + | |
| self.normalized_word_changes[key] ** 2 + | |
| self.normalized_kl_divergences[key] ** 2 + | |
| self.normalized_perplexities[key] ** 2 | |
| ) / 4 | |
| ) | |
| self.combined_distortions[key] = rms | |
| def plot_metrics(self): | |
| """ | |
| Plot each normalized metric and the combined distortion in separate graphs. | |
| """ | |
| import matplotlib.pyplot as plt | |
| keys = list(self.normalized_levenshtein.keys()) | |
| indices = np.arange(len(keys)) | |
| # Prepare data for plotting | |
| metrics = { | |
| 'Levenshtein Distance': [self.normalized_levenshtein[key] for key in keys], | |
| 'Word-Level Changes': [self.normalized_word_changes[key] for key in keys], | |
| 'KL Divergence': [self.normalized_kl_divergences[key] for key in keys], | |
| 'Perplexity': [self.normalized_perplexities[key] for key in keys], | |
| 'Combined Distortion': [self.combined_distortions[key] for key in keys] | |
| } | |
| # Plot each metric separately | |
| for metric_name, values in metrics.items(): | |
| plt.figure(figsize=(12, 6)) | |
| plt.plot(indices, values, marker='o', color=np.random.rand(3,)) | |
| plt.xlabel('Sentence Index') | |
| plt.ylabel('Normalized Value (0-1)') | |
| plt.title(f'Normalized {metric_name}') | |
| plt.grid(True) | |
| plt.tight_layout() | |
| plt.show() | |
| # Private methods for metric calculations | |
| def _calculate_levenshtein_distance(self, modified_sentence): | |
| """ | |
| Calculate the Levenshtein Distance between the original and modified sentence. | |
| """ | |
| return nltk.edit_distance(self.original_sentence, modified_sentence) | |
| def _calculate_word_level_change(self, modified_sentence): | |
| """ | |
| Calculate the proportion of word-level changes between the original and modified sentence. | |
| """ | |
| original_words = self.original_sentence.split() | |
| modified_words = modified_sentence.split() | |
| total_words = max(len(original_words), len(modified_words)) | |
| changed_words = sum(1 for o, m in zip(original_words, modified_words) if o != m) | |
| # Account for extra words in the modified sentence | |
| changed_words += abs(len(original_words) - len(modified_words)) | |
| distortion = changed_words / total_words | |
| return distortion | |
| def _calculate_kl_divergence(self, modified_sentence): | |
| """ | |
| Calculate the KL Divergence between the word distributions of the original and modified sentence. | |
| """ | |
| original_counts = Counter(self.original_sentence.lower().split()) | |
| modified_counts = Counter(modified_sentence.lower().split()) | |
| all_words = set(original_counts.keys()).union(set(modified_counts.keys())) | |
| original_probs = np.array([original_counts.get(word, 0) for word in all_words], dtype=float) | |
| modified_probs = np.array([modified_counts.get(word, 0) for word in all_words], dtype=float) | |
| # Add smoothing to avoid division by zero | |
| original_probs += 1e-10 | |
| modified_probs += 1e-10 | |
| # Normalize to create probability distributions | |
| original_probs /= original_probs.sum() | |
| modified_probs /= modified_probs.sum() | |
| kl_divergence = np.sum(rel_entr(original_probs, modified_probs)) | |
| return kl_divergence | |
| def _calculate_perplexity(self, sentence): | |
| """ | |
| Calculate the perplexity of a sentence using GPT-2. | |
| """ | |
| encodings = self.tokenizer(sentence, return_tensors='pt') | |
| max_length = self.model.config.n_positions | |
| stride = max_length | |
| lls = [] | |
| for i in range(0, encodings.input_ids.size(1), stride): | |
| begin_loc = i | |
| end_loc = min(i + stride, encodings.input_ids.size(1)) | |
| trg_len = end_loc - begin_loc | |
| input_ids = encodings.input_ids[:, begin_loc:end_loc] | |
| target_ids = input_ids.clone() | |
| with torch.no_grad(): | |
| outputs = self.model(input_ids, labels=target_ids) | |
| log_likelihood = outputs.loss * trg_len | |
| lls.append(log_likelihood) | |
| ppl = torch.exp(torch.stack(lls).sum() / end_loc) | |
| return ppl.item() | |
| def _normalize_dict(self, metric_dict): | |
| """ | |
| Normalize the values in a dictionary to be between 0 and 1. | |
| """ | |
| values = np.array(list(metric_dict.values())) | |
| min_val = values.min() | |
| max_val = values.max() | |
| # Avoid division by zero if all values are the same | |
| if max_val - min_val == 0: | |
| normalized_values = np.zeros_like(values) | |
| else: | |
| normalized_values = (values - min_val) / (max_val - min_val) | |
| return dict(zip(metric_dict.keys(), normalized_values)) | |
| # Getter methods | |
| def get_normalized_metrics(self): | |
| """ | |
| Get all normalized metrics as a dictionary. | |
| """ | |
| return { | |
| 'Levenshtein Distance': self.normalized_levenshtein, | |
| 'Word-Level Changes': self.normalized_word_changes, | |
| 'KL Divergence': self.normalized_kl_divergences, | |
| 'Perplexity': self.normalized_perplexities | |
| } | |
| def get_combined_distortions(self): | |
| """ | |
| Get the dictionary of combined distortion values. | |
| """ | |
| return self.combined_distortions | |
| # # Example usage | |
| # if __name__ == "__main__": | |
| # # Original sentence | |
| # original_sentence = "The quick brown fox jumps over the lazy dog" | |
| # paraphrased_sentences = [ | |
| # # Original 1: "A swift auburn fox leaps across a sleepy canine." | |
| # "The swift auburn fox leaps across a sleepy canine.", | |
| # "A quick auburn fox leaps across a sleepy canine.", | |
| # "A swift ginger fox leaps across a sleepy canine.", | |
| # "A swift auburn fox bounds across a sleepy canine.", | |
| # "A swift auburn fox leaps across a tired canine.", | |
| # "Three swift auburn foxes leap across a sleepy canine.", | |
| # "The vulpine specimen rapidly traverses over a dormant dog.", | |
| # "Like lightning, the russet hunter soars over the drowsy guardian.", | |
| # "Tha quick ginger fox jumps o'er the lazy hound, ye ken.", | |
| # "One rapid Vulpes vulpes traverses the path of a quiescent canine.", | |
| # "A swift auburn predator navigates across a lethargic pet.", | |
| # "Subject A (fox) demonstrates velocity over Subject B (dog).", | |
| # # Original 2: "The agile russet fox bounds over an idle hound." | |
| # "Some agile russet foxes bound over an idle hound.", | |
| # "The nimble russet fox bounds over an idle hound.", | |
| # "The agile brown fox bounds over an idle hound.", | |
| # "The agile russet fox jumps over an idle hound.", | |
| # "The agile russet fox bounds over a lazy hound.", | |
| # "Two agile russet foxes bound over an idle hound.", | |
| # "A dexterous vulpine surpasses a stationary canine.", | |
| # "Quick as thought, the copper warrior sails over the guardian.", | |
| # "Tha nimble reddish fox jumps o'er the doggo, don't ya know.", | |
| # "A dexterous V. vulpes exceeds the plane of an inactive canine.", | |
| # "An agile russet hunter maneuvers above a resting hound.", | |
| # "Test subject F-1 achieves displacement superior to subject D-1.", | |
| # # Original 3: "A nimble mahogany vulpine vaults above a drowsy dog." | |
| # "The nimble mahogany vulpine vaults above a drowsy dog.", | |
| # "A swift mahogany vulpine vaults above a drowsy dog.", | |
| # "A nimble reddish vulpine vaults above a drowsy dog.", | |
| # "A nimble mahogany fox vaults above a drowsy dog.", | |
| # "A nimble mahogany vulpine leaps above a drowsy dog.", | |
| # "Four nimble mahogany vulpines vault above a drowsy dog.", | |
| # "An agile specimen of reddish fur surpasses a somnolent canine.", | |
| # "Fleet as wind, the earth-toned hunter soars over the sleepy guard.", | |
| # "Tha quick brown beastie jumps o'er the tired pup, aye.", | |
| # "Single V. vulpes demonstrates vertical traverse over C. familiaris.", | |
| # "A nimble rust-colored predator crosses above a drowsy pet.", | |
| # "Observed: Subject Red executes vertical motion over Subject Gray.", | |
| # # Original 4: "The speedy copper-colored fox hops over the lethargic pup." | |
| # "A speedy copper-colored fox hops over the lethargic pup.", | |
| # "The quick copper-colored fox hops over the lethargic pup.", | |
| # "The speedy bronze fox hops over the lethargic pup.", | |
| # "The speedy copper-colored fox jumps over the lethargic pup.", | |
| # "The speedy copper-colored fox hops over the tired pup.", | |
| # "Multiple speedy copper-colored foxes hop over the lethargic pup.", | |
| # "A rapid vulpine of bronze hue traverses an inactive young canine.", | |
| # "Swift as a dart, the metallic hunter bounds over the lazy puppy.", | |
| # "Tha fast copper beastie leaps o'er the sleepy wee dog.", | |
| # "1 rapid V. vulpes crosses above 1 juvenile C. familiaris.", | |
| # "A fleet copper-toned predator moves past a sluggish young dog.", | |
| # "Field note: Adult fox subject exceeds puppy subject vertically.", | |
| # # Original 5: "A rapid tawny fox springs over a sluggish dog." | |
| # "The rapid tawny fox springs over a sluggish dog.", | |
| # "A quick tawny fox springs over a sluggish dog.", | |
| # "A rapid golden fox springs over a sluggish dog.", | |
| # "A rapid tawny fox jumps over a sluggish dog.", | |
| # "A rapid tawny fox springs over a lazy dog.", | |
| # "Six rapid tawny foxes spring over a sluggish dog.", | |
| # "An expeditious yellowish vulpine surpasses a torpid canine.", | |
| # "Fast as a bullet, the golden hunter vaults over the idle guard.", | |
| # "Tha swift yellowy fox jumps o'er the lazy mutt, aye.", | |
| # "One V. vulpes displays rapid transit over one inactive C. familiaris.", | |
| # "A speedy yellow-brown predator bypasses a motionless dog.", | |
| # "Log entry: Vulpine subject achieves swift vertical displacement.", | |
| # # Original 6: "The fleet-footed chestnut fox soars above an indolent canine." | |
| # "A fleet-footed chestnut fox soars above an indolent canine.", | |
| # "The swift chestnut fox soars above an indolent canine.", | |
| # "The fleet-footed brown fox soars above an indolent canine.", | |
| # "The fleet-footed chestnut fox leaps above an indolent canine.", | |
| # "The fleet-footed chestnut fox soars above a lazy canine.", | |
| # "Several fleet-footed chestnut foxes soar above an indolent canine.", | |
| # "A rapid brown vulpine specimen traverses a lethargic domestic dog.", | |
| # "Graceful as a bird, the nutbrown hunter flies over the lazy guard.", | |
| # "Tha quick brown beastie sails o'er the sleepy hound, ken.", | |
| # "Single agile V. vulpes achieves elevation above stationary canine.", | |
| # "A nimble brown predator glides over an unmoving domestic animal.", | |
| # "Research note: Brown subject displays superior vertical mobility.", | |
| # # Original 7: "A fast ginger fox hurdles past a slothful dog." | |
| # "The fast ginger fox hurdles past a slothful dog.", | |
| # "A quick ginger fox hurdles past a slothful dog.", | |
| # "A fast red fox hurdles past a slothful dog.", | |
| # "A fast ginger fox jumps past a slothful dog.", | |
| # "A fast ginger fox hurdles past a lazy dog.", | |
| # "Five fast ginger foxes hurdle past a slothful dog.", | |
| # "A rapid orange vulpine bypasses a lethargic canine.", | |
| # "Quick as lightning, the flame-colored hunter races past the lazy guard.", | |
| # "Tha swift ginger beastie leaps past the tired doggy, ye see.", | |
| # "1 rapid orange V. vulpes surpasses 1 inactive C. familiaris.", | |
| # "A speedy red-orange predator overtakes a motionless dog.", | |
| # "Data point: Orange subject demonstrates rapid transit past Gray subject.", | |
| # # Original 8: "The spry rusty-colored fox jumps across a dozing hound." | |
| # "A spry rusty-colored fox jumps across a dozing hound.", | |
| # "The agile rusty-colored fox jumps across a dozing hound.", | |
| # "The spry reddish fox jumps across a dozing hound.", | |
| # "The spry rusty-colored fox leaps across a dozing hound.", | |
| # "The spry rusty-colored fox jumps across a sleeping hound.", | |
| # "Multiple spry rusty-colored foxes jump across a dozing hound.", | |
| # "An agile rust-toned vulpine traverses a somnolent canine.", | |
| # "Nimble as thought, the copper hunter bounds over the resting guard.", | |
| # "Tha lively rust-colored beastie hops o'er the snoozin' hound.", | |
| # "Single dexterous V. vulpes crosses path of dormant C. familiaris.", | |
| # "A lithe rust-tinted predator moves past a slumbering dog.", | |
| # "Observation: Russet subject exhibits agility over dormant subject.", | |
| # # Original 9: "A quick tan fox leaps over an inactive dog." | |
| # "The quick tan fox leaps over an inactive dog.", | |
| # "A swift tan fox leaps over an inactive dog.", | |
| # "A quick beige fox leaps over an inactive dog.", | |
| # "A quick tan fox jumps over an inactive dog.", | |
| # "A quick tan fox leaps over a motionless dog.", | |
| # "Seven quick tan foxes leap over an inactive dog.", | |
| # "A rapid light-brown vulpine surpasses a stationary canine.", | |
| # "Fast as wind, the sand-colored hunter soars over the still guard.", | |
| # "Tha nimble tan beastie jumps o'er the quiet doggy, aye.", | |
| # "One agile fawn V. vulpes traverses one immobile C. familiaris.", | |
| # "A fleet tan-colored predator bypasses an unmoving dog.", | |
| # "Field report: Tan subject demonstrates movement over static subject.", | |
| # # Original 10: "The brisk auburn vulpine bounces over a listless canine." | |
| # "Some brisk auburn vulpines bounce over a listless canine.", | |
| # "The quick auburn vulpine bounces over a listless canine.", | |
| # "The brisk russet vulpine bounces over a listless canine.", | |
| # "The brisk auburn fox bounces over a listless canine.", | |
| # "The brisk auburn vulpine jumps over a listless canine.", | |
| # "Five brisk auburn vulpines bounce over a listless canine.", | |
| # "The expeditious specimen supersedes a quiescent Canis lupus.", | |
| # "Swift as wind, the russet hunter vaults over the idle guardian.", | |
| # "Tha quick ginger beastie hops o'er the lazy mutt, aye.", | |
| # "One V. vulpes achieves displacement over inactive C. familiaris.", | |
| # "A high-velocity auburn predator traverses an immobile animal.", | |
| # "Final observation: Red subject shows mobility over Gray subject." | |
| # ] | |
| # # Initialize the calculator | |
| # calculator = SentenceDistortionCalculator(original_sentence, paraphrased_sentences) | |
| # # Calculate all metrics | |
| # calculator.calculate_all_metrics() | |
| # # Normalize the metrics | |
| # calculator.normalize_metrics() | |
| # # Calculate combined distortion | |
| # calculator.calculate_combined_distortion() | |
| # # Retrieve the normalized metrics and combined distortions | |
| # normalized_metrics = calculator.get_normalized_metrics() | |
| # combined_distortions = calculator.get_combined_distortions() | |
| # distortion_val=combined_distortions | |
| # # Display the results | |
| # print("Normalized Metrics:") | |
| # for metric_name, metric_dict in normalized_metrics.items(): | |
| # print(f"\n{metric_name}:") | |
| # for key, value in metric_dict.items(): | |
| # print(f"{key}: {value:.4f}") | |
| # print("\nCombined Distortions:") | |
| # for key, value in combined_distortions.items(): | |
| # print(f"{key}: {value:.4f}") | |
| # # Plot the metrics | |
| # calculator.plot_metrics() | |