Upload folder using huggingface_hub

app.py

@@ -1,11 +1,8 @@
 import nltk
 nltk.download('stopwords')
-# from transformers import AutoTokenizer
-# from transformers import AutoModelForSeq2SeqLM
-import plotly.graph_objs as go
-from transformers import pipeline
 import random
 import gradio as gr
+import time
 from tree import generate_subplot1, generate_subplot2
 from paraphraser import generate_paraphrase
 from lcs import find_common_subsequences, find_common_gram_positions
@@ -17,196 +14,326 @@ from detectability import SentenceDetectabilityCalculator
 from distortion import SentenceDistortionCalculator
 from euclidean_distance import SentenceEuclideanDistanceCalculator
 from threeD_plot import gen_three_D_plot
-
-
-# Function for the Gradio interface
-def model(prompt):
-    user_prompt = prompt
-    paraphrased_sentences = generate_paraphrase(user_prompt)
-    analyzed_paraphrased_sentences, selected_sentences, discarded_sentences = analyze_entailment(user_prompt, paraphrased_sentences, 0.7)
-    print(analyze_entailment(user_prompt, paraphrased_sentences, 0.7))
-    common_grams = find_common_subsequences(user_prompt, selected_sentences)
-    subsequences = [subseq for _, subseq in common_grams]
-    common_grams_position = find_common_gram_positions(selected_sentences, subsequences)
-
-    masked_sentences = []
-    masked_words = []
-    masked_logits = []
-
-    for sentence in paraphrased_sentences:
-        masked_sent, logits, words = mask_non_stopword(sentence)
-        masked_sentences.append(masked_sent)
-        masked_words.append(words)
-        masked_logits.append(logits)
+from sankey import generate_sankey_diagram
+
+class WatermarkingPipeline:
+    def __init__(self):
+        # Existing initialization code...
+        self.user_prompt = None
+        self.paraphrased_sentences = None
+        self.analyzed_paraphrased_sentences = None
+        self.selected_sentences = None
+        self.discarded_sentences = None
+        self.common_grams = None
+        self.subsequences = None
+        self.common_grams_position = None
+        self.masked_sentences = None
+        self.masked_words = None
+        self.masked_logits = None
+        self.sampled_sentences = None
+        self.reparaphrased_sentences = None
+        self.distortion_list = None
+        self.detectability_list = None
+        self.euclidean_dist_list = None
+    
+    def step1_paraphrasing(self, prompt, threshold=0.7):
+        start_time = time.time()
         
-        masked_sent, logits, words = mask_non_stopword_pseudorandom(sentence)
-        masked_sentences.append(masked_sent)
-        masked_words.append(words)
-        masked_logits.append(logits)
+        # Existing step1 code...
+        self.user_prompt = prompt
+        self.paraphrased_sentences = generate_paraphrase(prompt)
+        if self.paraphrased_sentences is None:
+            return "Error in generating paraphrases", "Error: Could not complete step"
         
-        masked_sent, logits, words = high_entropy_words(sentence, common_grams)
-        masked_sentences.append(masked_sent)
-        masked_words.append(words)
-        masked_logits.append(logits)
-
-    sampled_sentences = []
-    for masked_sent, words, logits in zip(masked_sentences, masked_words, masked_logits):
-        sampled_sentences.append(sample_word(masked_sent, words, logits, sampling_technique='inverse_transform', temperature=1.0))
-        sampled_sentences.append(sample_word(masked_sent, words, logits, sampling_technique='exponential_minimum', temperature=1.0))
-        sampled_sentences.append(sample_word(masked_sent, words, logits, sampling_technique='temperature', temperature=1.0))
-        sampled_sentences.append(sample_word(masked_sent, words, logits, sampling_technique='greedy', temperature=1.0))
-
-
-
-
-    colors = ["red", "blue", "brown", "green"]
-
-    def select_color():
-        return random.choice(colors)
-
-    highlight_info = [(word, select_color()) for _, word in common_grams]
-
-    highlighted_user_prompt = highlight_common_words(common_grams, [user_prompt], "Non-melting Points in the User Prompt")
-    highlighted_accepted_sentences = highlight_common_words_dict(common_grams, selected_sentences, "Paraphrased Sentences")
-    highlighted_discarded_sentences = highlight_common_words_dict(common_grams, discarded_sentences, "Discarded Sentences")
-
-    trees1 = []
-    trees2 = []
-
-    masked_index = 0
-    sampled_index = 0
-
-    for i, sentence in enumerate(paraphrased_sentences):
-        next_masked_sentences = masked_sentences[masked_index:masked_index + 3]
-        next_sampled_sentences = sampled_sentences[sampled_index:sampled_index + 12]
-
-        tree1 = generate_subplot1(sentence, next_masked_sentences, highlight_info, common_grams)
-        trees1.append(tree1)
-
-        tree2 = generate_subplot2(next_masked_sentences, next_sampled_sentences, highlight_info, common_grams)
-        trees2.append(tree2)
-
-        masked_index += 3 
-        sampled_index += 12
-
-    reparaphrased_sentences = generate_paraphrase(sampled_sentences)
-
-    len_reparaphrased_sentences = len(reparaphrased_sentences)
-
-    reparaphrased_sentences_list = []
-
-    # Process the sentences in batches of 10
-    for i in range(0, len_reparaphrased_sentences, 10):
-        # Get the current batch of 10 sentences
-        batch = reparaphrased_sentences[i:i + 10]
+        self.analyzed_paraphrased_sentences, self.selected_sentences, self.discarded_sentences = \
+            analyze_entailment(self.user_prompt, self.paraphrased_sentences, threshold)
         
-        # Check if the batch has exactly 10 sentences
-        if len(batch) == 10:
-            # Call the display_sentences function and store the result in the list
-            html_block = reparaphrased_sentences_html(batch)
-            reparaphrased_sentences_list.append(html_block)
-
-    distortion_list = []
-    detectability_list = []
-    euclidean_dist_list = []
-
-    distortion_calculator = SentenceDistortionCalculator(user_prompt, reparaphrased_sentences)
-    distortion_calculator.calculate_all_metrics()
-    distortion_calculator.normalize_metrics()
-    distortion_calculator.calculate_combined_distortion()
-
-    distortion = distortion_calculator.get_combined_distortions()
-
-    for each in distortion.items():
-        distortion_list.append(each[1])
-
-    detectability_calculator = SentenceDetectabilityCalculator(user_prompt, reparaphrased_sentences)
-    detectability_calculator.calculate_all_metrics()
-    detectability_calculator.normalize_metrics()
-    detectability_calculator.calculate_combined_detectability()
-
-    detectability = detectability_calculator.get_combined_detectabilities()
-
-    for each in detectability.items():
-        detectability_list.append(each[1])
-
-    euclidean_dist_calculator = SentenceEuclideanDistanceCalculator(user_prompt, reparaphrased_sentences)
-    euclidean_dist_calculator.calculate_all_metrics()
-    euclidean_dist_calculator.normalize_metrics()
-    euclidean_dist_calculator.get_normalized_metrics()
-
-    euclidean_dist = detectability_calculator.get_combined_detectabilities()
-
-    for each in euclidean_dist.items():
-        euclidean_dist_list.append(each[1])
-
-    three_D_plot = gen_three_D_plot(detectability_list, distortion_list, euclidean_dist_list)
-
-    return [highlighted_user_prompt, highlighted_accepted_sentences, highlighted_discarded_sentences] + trees1 + trees2 + reparaphrased_sentences_list + [three_D_plot]
-
+        self.common_grams = find_common_subsequences(self.user_prompt, self.selected_sentences)
+        self.subsequences = [subseq for _, subseq in self.common_grams]
+        self.common_grams_position = find_common_gram_positions(self.selected_sentences, self.subsequences)
+        
+        colors = ["red", "blue", "brown", "green"]
+        def select_color():
+            return random.choice(colors)
+        highlight_info = [(word, select_color()) for _, word in self.common_grams]
+        
+        highlighted_user_prompt = highlight_common_words(
+            self.common_grams, [self.user_prompt], "Highlighted LCS in the User Prompt"
+        )
+        highlighted_accepted_sentences = highlight_common_words_dict(
+            self.common_grams, self.selected_sentences, "Paraphrased Sentences"
+        )
+        highlighted_discarded_sentences = highlight_common_words_dict(
+            self.common_grams, self.discarded_sentences, "Discarded Sentences"
+        )
+        
+        execution_time = time.time() - start_time
+        time_info = f"Step 1 completed in {execution_time:.2f} seconds"
+        
+        return [
+            highlighted_user_prompt, 
+            highlighted_accepted_sentences, 
+            highlighted_discarded_sentences,
+            time_info
+        ]
+        
+    def step2_masking(self):
+        start_time = time.time()
+        
+        if self.paraphrased_sentences is None:
+            return [None] * 10 + ["Error: Please complete step 1 first"]
+
+        # Existing step2 code...
+        self.masked_sentences = []
+        self.masked_words = []
+        self.masked_logits = []
+
+        for sentence in self.paraphrased_sentences:
+            for mask_func in [mask_non_stopword, mask_non_stopword_pseudorandom, 
+                            lambda s: high_entropy_words(s, self.common_grams)]:
+                masked_sent, logits, words = mask_func(sentence)
+                self.masked_sentences.append(masked_sent)
+                self.masked_words.append(words)
+                self.masked_logits.append(logits)
+
+        trees = []
+        masked_index = 0
+        colors = ["red", "blue", "brown", "green"]
+        highlight_info = [(word, random.choice(colors)) for _, word in self.common_grams]
+
+        for i, sentence in enumerate(self.paraphrased_sentences):
+            next_masked = self.masked_sentences[masked_index:masked_index + 3]
+            tree = generate_subplot1(sentence, next_masked, highlight_info, self.common_grams)
+            trees.append(tree)
+            masked_index += 3
+
+        execution_time = time.time() - start_time
+        time_info = f"Step 2 completed in {execution_time:.2f} seconds"
+        
+        return trees + [time_info]
 
-with gr.Blocks(theme=gr.themes.Monochrome()) as demo:
-    gr.Markdown("# **AIISC Watermarking Model**")
+    def step3_sampling(self):
+        start_time = time.time()
+        
+        if self.masked_sentences is None:
+            return [None] * 10 + ["Error: Please complete step 2 first"]
+
+        # Existing step3 code...
+        self.sampled_sentences = []
+        trees = []
+        colors = ["red", "blue", "brown", "green"]
+        highlight_info = [(word, random.choice(colors)) for _, word in self.common_grams]
+
+        sampling_techniques = [
+            ('inverse_transform', 1.0),
+            ('exponential_minimum', 1.0),
+            ('temperature', 1.0),
+            ('greedy', 1.0)
+        ]
+
+        masked_index = 0
+        while masked_index < len(self.masked_sentences):
+            current_masked = self.masked_sentences[masked_index:masked_index + 3]
+            current_words = self.masked_words[masked_index:masked_index + 3]
+            current_logits = self.masked_logits[masked_index:masked_index + 3]
+            
+            batch_samples = []
+            for masked_sent, words, logits in zip(current_masked, current_words, current_logits):
+                for technique, temp in sampling_techniques:
+                    sampled = sample_word(masked_sent, words, logits, 
+                                       sampling_technique=technique, 
+                                       temperature=temp)
+                    batch_samples.append(sampled)
+            
+            self.sampled_sentences.extend(batch_samples)
+            
+            if current_masked:
+                tree = generate_subplot2(
+                    current_masked,
+                    batch_samples,
+                    highlight_info,
+                    self.common_grams
+                )
+                trees.append(tree)
+            
+            masked_index += 3
+
+        if len(trees) < 10:
+            trees.extend([None] * (10 - len(trees)))
+            
+        execution_time = time.time() - start_time
+        time_info = f"Step 3 completed in {execution_time:.2f} seconds"
+        
+        return trees[:10] + [time_info]
+    
+    def step4_reparaphrase(self):
+        start_time = time.time()
+        
+        if self.sampled_sentences is None:
+            return ["Error: Please complete step 3 first"] * 120 + ["Error: Please complete step 3 first"]
+        
+        # Existing step4 code...
+        self.reparaphrased_sentences = []
+        for i in range(13):
+            self.reparaphrased_sentences.append(generate_paraphrase(self.sampled_sentences[i]))
+            
+        reparaphrased_sentences_list = []
+        for i in range(0, len(self.reparaphrased_sentences), 10):
+            batch = self.reparaphrased_sentences[i:i + 10]
+            if len(batch) == 10:
+                html_block = reparaphrased_sentences_html(batch)
+                reparaphrased_sentences_list.append(html_block)
+                
+        execution_time = time.time() - start_time
+        time_info = f"Step 4 completed in {execution_time:.2f} seconds"
+        
+        return reparaphrased_sentences_list + [time_info]
+    
+    def step5_metrics(self):
+        start_time = time.time()
+        
+        if self.reparaphrased_sentences is None:
+            return "Please complete step 4 first", "Error: Please complete step 4 first"
+        
+        # Existing step5 code...
+        distortion_calculator = SentenceDistortionCalculator(self.user_prompt, self.reparaphrased_sentences)
+        distortion_calculator.calculate_all_metrics()
+        distortion_calculator.normalize_metrics()
+        distortion_calculator.calculate_combined_distortion()
+        distortion = distortion_calculator.get_combined_distortions()
+        self.distortion_list = [each[1] for each in distortion.items()]
+        
+        detectability_calculator = SentenceDetectabilityCalculator(self.user_prompt, self.reparaphrased_sentences)
+        detectability_calculator.calculate_all_metrics()
+        detectability_calculator.normalize_metrics()
+        detectability_calculator.calculate_combined_detectability()
+        detectability = detectability_calculator.get_combined_detectabilities()
+        self.detectability_list = [each[1] for each in detectability.items()]
+        
+        euclidean_dist_calculator = SentenceEuclideanDistanceCalculator(self.user_prompt, self.reparaphrased_sentences)
+        euclidean_dist_calculator.calculate_all_metrics()
+        euclidean_dist_calculator.normalize_metrics()
+        euclidean_dist = detectability_calculator.get_combined_detectabilities()
+        self.euclidean_dist_list = [each[1] for each in euclidean_dist.items()]
+        
+        three_D_plot = gen_three_D_plot(
+            self.detectability_list, 
+            self.distortion_list, 
+            self.euclidean_dist_list
+        )
+        
+        execution_time = time.time() - start_time
+        time_info = f"Step 5 completed in {execution_time:.2f} seconds"
+        
+        return three_D_plot, time_info
 
-    with gr.Row():
-        user_input = gr.Textbox(label="User Prompt")
+    def step6_sankey(self):
+        return generate_sankey_diagram()
 
-    with gr.Row():
-        submit_button = gr.Button("Submit")
-        clear_button = gr.Button("Clear")
+def create_gradio_interface():
+    pipeline = WatermarkingPipeline()
+    
+    with gr.Blocks(theme=gr.themes.Monochrome()) as demo:
+        gr.Markdown("# **AIISC Watermarking Model**")
+        
+        with gr.Column():
+            gr.Markdown("## Input Prompt")
+            user_input = gr.Textbox(label="Enter Your Prompt")
 
-    with gr.Row():
-        highlighted_user_prompt = gr.HTML()
+        gr.Markdown("## Step 1: Paraphrasing, LCS and Entailment Analysis")
+        paraphrase_button = gr.Button("Generate Paraphrases")
+        highlighted_user_prompt = gr.HTML(label="Highlighted User Prompt")
 
-    with gr.Row():
         with gr.Tabs():
-            with gr.TabItem("Paraphrased Sentences"):
+            with gr.TabItem("Accepted Paraphrased Sentences"):
                 highlighted_accepted_sentences = gr.HTML()
-            with gr.TabItem("Discarded Sentences"):
+            with gr.TabItem("Discarded Paraphrased Sentences"):
                 highlighted_discarded_sentences = gr.HTML()
-
-    # Adding labels before the tree plots
-    with gr.Row():
-        gr.Markdown("### Where to Watermark?")  # Label for masked sentences trees
-    with gr.Row():
+        step1_time = gr.Textbox(label="Execution Time", interactive=False)
+            
+        gr.Markdown("## Step 2: Where to Mask?")
+        masking_button = gr.Button("Apply Masking")
+        gr.Markdown("### Masked Sentence Trees")
         with gr.Tabs():
             tree1_tabs = []
-            for i in range(10):  # Adjust this range according to the number of trees
-                with gr.TabItem(f"Sentence {i+1}"):
+            for i in range(10):
+                with gr.TabItem(f"Masked Sentence {i+1}"):
                     tree1 = gr.Plot()
                     tree1_tabs.append(tree1)
-
-    with gr.Row():
-        gr.Markdown("### How to Watermark?")  # Label for sampled sentences trees
-    with gr.Row():
+        step2_time = gr.Textbox(label="Execution Time", interactive=False)
+            
+        gr.Markdown("## Step 3: How to Mask?")
+        sampling_button = gr.Button("Sample Words")
+        gr.Markdown("### Sampled Sentence Trees")
         with gr.Tabs():
             tree2_tabs = []
-            for i in range(10):  # Adjust this range according to the number of trees
-                with gr.TabItem(f"Sentence {i+1}"):
+            for i in range(10):
+                with gr.TabItem(f"Sampled Sentence {i+1}"):
                     tree2 = gr.Plot()
                     tree2_tabs.append(tree2)
-
-        # Adding the "Re-paraphrased Sentences" section
-    with gr.Row():
-        gr.Markdown("### Re-paraphrased Sentences")  # Label for re-paraphrased sentences
-
-    # Adding tabs for the re-paraphrased sentences
-    with gr.Row():
+        step3_time = gr.Textbox(label="Execution Time", interactive=False)
+        
+        gr.Markdown("## Step 4: Re-paraphrasing")
+        reparaphrase_button = gr.Button("Re-paraphrase")
+        gr.Markdown("### Reparaphrased Sentences")
         with gr.Tabs():
             reparaphrased_sentences_tabs = []
-            for i in range(120):  # 120 tabs for 120 batches of sentences
-                with gr.TabItem(f"Sentence {i+1}"):
-                    reparaphrased_sent_html = gr.HTML()  # Placeholder for each batch
+            for i in range(120):
+                with gr.TabItem(f"Reparaphrased Batch {i+1}"):
+                    reparaphrased_sent_html = gr.HTML()
                     reparaphrased_sentences_tabs.append(reparaphrased_sent_html)
-
-    with gr.Row():
-        gr.Markdown("### 3D Plot for Sweet Spot")
-    with gr.Row():
+        step4_time = gr.Textbox(label="Execution Time", interactive=False)
+        
+        gr.Markdown("## Step 5: Finding Sweet Spot")
+        metrics_button = gr.Button("Calculate Metrics")
+        gr.Markdown("### 3D Visualization of Metrics")
         three_D_plot = gr.Plot()
+        step5_time = gr.Textbox(label="Execution Time", interactive=False)
 
-
-    submit_button.click(model, inputs=user_input, outputs=[highlighted_user_prompt, highlighted_accepted_sentences, highlighted_discarded_sentences] + tree1_tabs + tree2_tabs + reparaphrased_sentences_tabs + [three_D_plot])
-    clear_button.click(lambda: "", inputs=None, outputs=user_input)
-    clear_button.click(lambda: "", inputs=None, outputs=[highlighted_user_prompt, highlighted_accepted_sentences, highlighted_discarded_sentences] + tree1_tabs + tree2_tabs + reparaphrased_sentences_tabs + [three_D_plot])
-
-demo.launch(share=True)
+        # Sankey Diagram
+        gr.Markdown("# Watermarking Pipeline Flow Visualization")
+        generate_button = gr.Button("Generate Sankey Diagram")
+        sankey_plot = gr.Plot()
+        
+        paraphrase_button.click(
+            pipeline.step1_paraphrasing, 
+            inputs=user_input, 
+            outputs=[highlighted_user_prompt, highlighted_accepted_sentences, highlighted_discarded_sentences, step1_time]
+        )
+        
+        masking_button.click(
+            pipeline.step2_masking, 
+            inputs=None, 
+            outputs=tree1_tabs + [step2_time]
+        )
+
+        sampling_button.click(
+            pipeline.step3_sampling, 
+            inputs=None, 
+            outputs=tree2_tabs + [step3_time],
+            show_progress=True
+        )
+        
+        reparaphrase_button.click(
+            pipeline.step4_reparaphrase, 
+            inputs=None, 
+            outputs=reparaphrased_sentences_tabs + [step4_time]
+        )
+        
+        metrics_button.click(
+            pipeline.step5_metrics, 
+            inputs=None, 
+            outputs=[three_D_plot, step5_time]
+        )
+
+        generate_button.click(
+            pipeline.step6_sankey,
+            inputs=None,
+            outputs=sankey_plot
+        )
+    
+    return demo
+
+if __name__ == "__main__":
+    demo = create_gradio_interface()
+    demo.launch(share=True)

detectability.py

@@ -6,12 +6,12 @@ import torch
 import matplotlib.pyplot as plt
 from sklearn.metrics.pairwise import cosine_similarity
 from transformers import BertModel, BertTokenizer
-from sentence_transformers import SentenceTransformer
-from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
+from scipy import stats
 
 # Download NLTK data if not already present
 nltk.download('punkt', quiet=True)
-detectability_val={}
+detectability_val = {}
+
 class SentenceDetectabilityCalculator:
     """
     A class to calculate and analyze detectability metrics between an original sentence and paraphrased sentences.
@@ -25,63 +25,62 @@ class SentenceDetectabilityCalculator:
         self.paraphrased_sentences = paraphrased_sentences
 
         # Raw metric dictionaries
-        self.bleu_scores = {}
-        self.cosine_similarities = {}
-        self.sts_scores = {}
+        self.z_scores = {}
+        self.p_values = {}
+        self.metric_values = []
 
         # Normalized metric dictionaries
-        self.normalized_bleu = {}
-        self.normalized_cosine = {}
-        self.normalized_sts = {}
+        self.normalized_z_scores = {}
+        self.normalized_p_values = {}
 
         # Combined detectability dictionary
         self.combined_detectabilities = {}
 
-        # Load pre-trained BERT and SentenceTransformer for Cosine Similarity and STS Score
+        # Load pre-trained BERT for embeddings
         self.bert_model = BertModel.from_pretrained('bert-base-uncased')
         self.bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-        self.sts_model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
 
     def calculate_all_metrics(self):
         """
-        Calculate all detectability metrics for each paraphrased sentence.
+        Calculate detectability metrics for each paraphrased sentence.
         """
         original_embedding = self._get_sentence_embedding(self.original_sentence)
-        sts_original_embedding = self.sts_model.encode(self.original_sentence)
 
+        # First, compute the metric values (cosine similarities)
         for idx, paraphrased_sentence in enumerate(self.paraphrased_sentences):
-            key = f"Sentence_{idx+1}"
-
-            # BLEU Score
-            self.bleu_scores[key] = self._calculate_bleu(self.original_sentence, paraphrased_sentence)
-
-            # Cosine Similarity
             paraphrase_embedding = self._get_sentence_embedding(paraphrased_sentence)
-            self.cosine_similarities[key] = cosine_similarity([original_embedding], [paraphrase_embedding])[0][0]
+            cosine_sim = cosine_similarity([original_embedding], [paraphrase_embedding])[0][0]
+            self.metric_values.append(cosine_sim)
+
+        # Compute mean and standard deviation of the metric values
+        metric_mean = np.mean(self.metric_values)
+        metric_std = np.std(self.metric_values)
 
-            # STS Score
-            sts_paraphrase_embedding = self.sts_model.encode(paraphrased_sentence)
-            self.sts_scores[key] = cosine_similarity([sts_original_embedding], [sts_paraphrase_embedding])[0][0]
+        # Compute z-scores and p-values
+        for idx, (paraphrased_sentence, metric_value) in enumerate(zip(self.paraphrased_sentences, self.metric_values)):
+            key = f"Sentence_{idx+1}"
+            z_score = (metric_value - metric_mean) / metric_std if metric_std != 0 else 0.0
+            p_value = stats.norm.sf(abs(z_score)) * 2  # two-tailed p-value
+            self.z_scores[key] = z_score
+            self.p_values[key] = p_value
 
     def normalize_metrics(self):
         """
-        Normalize all metrics to be between 0 and 1.
+        Normalize z-scores and p-values to be between 0 and 1.
         """
-        self.normalized_bleu = self._normalize_dict(self.bleu_scores)
-        self.normalized_cosine = self._normalize_dict(self.cosine_similarities)
-        self.normalized_sts = self._normalize_dict(self.sts_scores)
+        self.normalized_z_scores = self._normalize_dict(self.z_scores)
+        self.normalized_p_values = self._normalize_dict(self.p_values)
 
     def calculate_combined_detectability(self):
         """
         Calculate the combined detectability using the root mean square of the normalized metrics.
         """
-        for key in self.normalized_bleu.keys():
+        for key in self.normalized_z_scores.keys():
             rms = np.sqrt(
                 (
-                    self.normalized_bleu[key] ** 2 +
-                    self.normalized_cosine[key] ** 2 +
-                    self.normalized_sts[key] ** 2
-                ) / 3
+                    self.normalized_z_scores[key] ** 2 +
+                    self.normalized_p_values[key] ** 2
+                ) / 2
             )
             self.combined_detectabilities[key] = rms
 
@@ -89,14 +88,13 @@ class SentenceDetectabilityCalculator:
         """
         Plot each normalized metric and the combined detectability in separate graphs.
         """
-        keys = list(self.normalized_bleu.keys())
+        keys = list(self.normalized_z_scores.keys())
         indices = np.arange(len(keys))
 
         # Prepare data for plotting
         metrics = {
-            'BLEU Score': [self.normalized_bleu[key] for key in keys],
-            'Cosine Similarity': [self.normalized_cosine[key] for key in keys],
-            'STS Score': [self.normalized_sts[key] for key in keys],
+            'Z-Score': [self.normalized_z_scores[key] for key in keys],
+            'P-Value': [self.normalized_p_values[key] for key in keys],
             'Combined Detectability': [self.combined_detectabilities[key] for key in keys]
         }
 
@@ -111,16 +109,7 @@ class SentenceDetectabilityCalculator:
             plt.tight_layout()
             plt.show()
 
-    # Private methods for metric calculations
-    def _calculate_bleu(self, reference, candidate):
-        """
-        Calculate the BLEU score between the original and paraphrased sentence using smoothing.
-        """
-        reference_tokens = nltk.word_tokenize(reference)
-        candidate_tokens = nltk.word_tokenize(candidate)
-        smoothing = SmoothingFunction().method1
-        return sentence_bleu([reference_tokens], candidate_tokens, smoothing_function=smoothing)
-
+    # Private methods
     def _get_sentence_embedding(self, sentence):
         """
         Get sentence embedding using BERT.
@@ -150,9 +139,8 @@ class SentenceDetectabilityCalculator:
         Get all normalized metrics as a dictionary.
         """
         return {
-            'BLEU Score': self.normalized_bleu,
-            'Cosine Similarity': self.normalized_cosine,
-            'STS Score': self.normalized_sts
+            'Z-Score': self.normalized_z_scores,
+            'P-Value': self.normalized_p_values
         }
 
     def get_combined_detectabilities(self):
@@ -310,7 +298,6 @@ if __name__ == "__main__":
     "Final observation: Red subject shows mobility over Gray subject."
     ]
 
-
     # Initialize the calculator
     calculator = SentenceDetectabilityCalculator(original_sentence, paraphrased_sentences)
 
@@ -326,18 +313,12 @@ if __name__ == "__main__":
     # Retrieve the normalized metrics and combined detectabilities
     normalized_metrics = calculator.get_normalized_metrics()
     combined_detectabilities = calculator.get_combined_detectabilities()
-    detectability_val=combined_detectabilities
+    detectability_val = combined_detectabilities
 
     # Display the results
-    # print("Normalized Metrics:")
-    # for metric_name, metric_dict in normalized_metrics.items():
-    #     print(f"\n{metric_name}:")
-    #     for key, value in metric_dict.items():
-    #         print(f"{key}: {value:.4f}")
-
     print("\nCombined Detectabilities:")
     for each in combined_detectabilities.items():
         print(f"{each[1]}")
 
-    # Plot the metrics
-    # calculator.plot_metrics()
+    # Plot the metrics (optional)
+    #calculator.plot_metrics()

paraphraser.py

@@ -1,32 +1,32 @@
-# from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 
-# # Function to Initialize the Model
-# def init_model():
-#     para_tokenizer = AutoTokenizer.from_pretrained("humarin/chatgpt_paraphraser_on_T5_base")
-#     para_model = AutoModelForSeq2SeqLM.from_pretrained("humarin/chatgpt_paraphraser_on_T5_base")
-#     return para_tokenizer, para_model
+# Function to Initialize the Model
+def init_model():
+    para_tokenizer = AutoTokenizer.from_pretrained("humarin/chatgpt_paraphraser_on_T5_base")
+    para_model = AutoModelForSeq2SeqLM.from_pretrained("humarin/chatgpt_paraphraser_on_T5_base")
+    return para_tokenizer, para_model
 
-# # Function to Paraphrase the Text
-# def paraphrase(question, para_tokenizer, para_model, num_beams=10, num_beam_groups=10, num_return_sequences=10, repetition_penalty=10.0, diversity_penalty=3.0, no_repeat_ngram_size=2, temperature=0.7, max_length=64):
-#     input_ids = para_tokenizer(
-#         f'paraphrase: {question}',
-#         return_tensors="pt", padding="longest",
-#         max_length=max_length,
-#         truncation=True,
-#     ).input_ids
-#     outputs = para_model.generate(
-#         input_ids, temperature=temperature, repetition_penalty=repetition_penalty,
-#         num_return_sequences=num_return_sequences, no_repeat_ngram_size=no_repeat_ngram_size,
-#         num_beams=num_beams, num_beam_groups=num_beam_groups,
-#         max_length=max_length, diversity_penalty=diversity_penalty
-#     )
-#     res = para_tokenizer.batch_decode(outputs, skip_special_tokens=True)
-#     return res
+# Function to Paraphrase the Text
+def paraphrase(question, para_tokenizer, para_model, num_beams=10, num_beam_groups=10, num_return_sequences=10, repetition_penalty=10.0, diversity_penalty=3.0, no_repeat_ngram_size=2, temperature=0.7, max_length=64):
+    input_ids = para_tokenizer(
+        f'paraphrase: {question}',
+        return_tensors="pt", padding="longest",
+        max_length=max_length,
+        truncation=True,
+    ).input_ids
+    outputs = para_model.generate(
+        input_ids, temperature=temperature, repetition_penalty=repetition_penalty,
+        num_return_sequences=num_return_sequences, no_repeat_ngram_size=no_repeat_ngram_size,
+        num_beams=num_beams, num_beam_groups=num_beam_groups,
+        max_length=max_length, diversity_penalty=diversity_penalty
+    )
+    res = para_tokenizer.batch_decode(outputs, skip_special_tokens=True)
+    return res
 
-# def generate_paraphrase(question):
-#     para_tokenizer, para_model = init_model()
-#     res = paraphrase(question, para_tokenizer, para_model)
-#     return res
+def generate_paraphrase(question):
+    para_tokenizer, para_model = init_model()
+    res = paraphrase(question, para_tokenizer, para_model)
+    return res
 
 # print(generate_paraphrase("Donald Trump said at a campaign rally event in Wilkes-Barre, Pennsylvania, that there has “never been a more dangerous time 5since the Holocaust” to be Jewish in the United States."))
 
@@ -34,50 +34,50 @@
 Accepts a sentence or list of sentences and returns a lit of all their paraphrases using GPT-4.
 '''
 
-from openai import OpenAI
-from dotenv import load_dotenv
-load_dotenv()
-import os
+# from openai import OpenAI
+# from dotenv import load_dotenv
+# load_dotenv()
+# import os
 
-key = os.getenv("OPENAI_API_KEY")
+# key = os.getenv("OPENAI_API_KEY")
 
-# Initialize the OpenAI client
-client = OpenAI(
-    api_key=key  # Replace with your actual API key
-)
+# # Initialize the OpenAI client
+# client = OpenAI(
+#     api_key=key  # Replace with your actual API key
+# )
 
-# Function to paraphrase sentences using GPT-4
-def generate_paraphrase(sentences, model="gpt-4o", num_paraphrases=10, max_tokens=150, temperature=0.7):
-    # Ensure sentences is a list even if a single sentence is passed
-    if isinstance(sentences, str):
-        sentences = [sentences]
+# # Function to paraphrase sentences using GPT-4
+# def generate_paraphrase(sentences, model="gpt-4o", num_paraphrases=10, max_tokens=150, temperature=0.7):
+#     # Ensure sentences is a list even if a single sentence is passed
+#     if isinstance(sentences, str):
+#         sentences = [sentences]
 
-    paraphrased_sentences_list = []
+#     paraphrased_sentences_list = []
 
-    for sentence in sentences:
-        full_prompt = f"Paraphrase the following text: '{sentence}'"
-        try:
-            chat_completion = client.chat.completions.create(
-                messages=[
-                    {
-                        "role": "user",
-                        "content": full_prompt,
-                    }
-                ],
-                model=model,
-                max_tokens=max_tokens,
-                temperature=temperature,
-                n=num_paraphrases  # Number of paraphrased sentences to generate
-            )
-            # Extract the paraphrased sentences from the response
-            paraphrased_sentences = [choice.message.content.strip() for choice in chat_completion.choices]
-            # Append paraphrased sentences to the list
-            paraphrased_sentences_list.extend(paraphrased_sentences)
-        except Exception as e:
-            print(f"Error paraphrasing sentence '{sentence}': {e}")
+#     for sentence in sentences:
+#         full_prompt = f"Paraphrase the following text: '{sentence}'"
+#         try:
+#             chat_completion = client.chat.completions.create(
+#                 messages=[
+#                     {
+#                         "role": "user",
+#                         "content": full_prompt,
+#                     }
+#                 ],
+#                 model=model,
+#                 max_tokens=max_tokens,
+#                 temperature=temperature,
+#                 n=num_paraphrases  # Number of paraphrased sentences to generate
+#             )
+#             # Extract the paraphrased sentences from the response
+#             paraphrased_sentences = [choice.message.content.strip() for choice in chat_completion.choices]
+#             # Append paraphrased sentences to the list
+#             paraphrased_sentences_list.extend(paraphrased_sentences)
+#         except Exception as e:
+#             print(f"Error paraphrasing sentence '{sentence}': {e}")
     
-    return paraphrased_sentences_list
+#     return paraphrased_sentences_list
 
-result = generate_paraphrase("Mayor Eric Adams did not attend the first candidate forum for the New York City mayoral race, but his record — and the criminal charges he faces — received plenty of attention on Saturday from the Democrats who are running to unseat him.")
+# result = generate_paraphrase("Mayor Eric Adams did not attend the first candidate forum for the New York City mayoral race, but his record — and the criminal charges he faces — received plenty of attention on Saturday from the Democrats who are running to unseat him.")
 
-print(len(result))
+# print(len(result))

sankey.py

@@ -0,0 +1,86 @@
+import plotly.graph_objects as go
+
+def generate_sankey_diagram():
+    pipeline_metrics = {
+        'masking_methods': ['random masking', 'pseudorandom masking', 'high-entropy masking'],
+        'sampling_methods': ['inverse_transform sampling', 'exponential_minimum sampling', 'temperature sampling', 'greedy sampling'],
+        'scores': {
+            ('random masking', 'inverse_transform sampling'): {'detectability': 0.8, 'distortion': 0.2},
+            ('random masking', 'exponential_minimum sampling'): {'detectability': 0.7, 'distortion': 0.3},
+            ('random masking', 'temperature sampling'): {'detectability': 0.6, 'distortion': 0.4},
+            ('random masking', 'greedy sampling'): {'detectability': 0.5, 'distortion': 0.5},
+            ('pseudorandom masking', 'inverse_transform sampling'): {'detectability': 0.75, 'distortion': 0.25},
+            ('pseudorandom masking', 'exponential_minimum sampling'): {'detectability': 0.65, 'distortion': 0.35},
+            ('pseudorandom masking', 'temperature sampling'): {'detectability': 0.55, 'distortion': 0.45},
+            ('pseudorandom masking', 'greedy sampling'): {'detectability': 0.45, 'distortion': 0.55},
+            ('high-entropy masking', 'inverse_transform sampling'): {'detectability': 0.85, 'distortion': 0.15},
+            ('high-entropy masking', 'exponential_minimum sampling'): {'detectability': 0.75, 'distortion': 0.25},
+            ('high-entropy masking', 'temperature sampling'): {'detectability': 0.65, 'distortion': 0.35},
+            ('high-entropy masking', 'greedy sampling'): {'detectability': 0.55, 'distortion': 0.45}
+        }
+    }
+    
+    # Find best combination
+    best_score = 0
+    best_combo = None
+    for combo, metrics in pipeline_metrics['scores'].items():
+        score = metrics['detectability'] * (1 - metrics['distortion'])
+        if score > best_score:
+            best_score = score
+            best_combo = combo
+
+    label_list = ['Input'] + pipeline_metrics['masking_methods'] + pipeline_metrics['sampling_methods'] + ['Output']
+    
+    source = []
+    target = []
+    value = []
+    colors = [] 
+    
+    # Input to masking methods
+    for i in range(len(pipeline_metrics['masking_methods'])):
+        source.append(0)
+        target.append(i + 1)
+        value.append(1)
+        colors.append('rgba(0,0,255,0.2)' if pipeline_metrics['masking_methods'][i] != best_combo[0] else 'rgba(255,0,0,0.8)')
+
+    # Masking to sampling methods
+    sampling_start = len(pipeline_metrics['masking_methods']) + 1
+    for i, mask in enumerate(pipeline_metrics['masking_methods']):
+        for j, sample in enumerate(pipeline_metrics['sampling_methods']):
+            score = pipeline_metrics['scores'][(mask, sample)]['detectability'] * \
+                   (1 - pipeline_metrics['scores'][(mask, sample)]['distortion'])
+            source.append(i + 1)
+            target.append(sampling_start + j)
+            value.append(score)
+            colors.append('rgba(0,0,255,0.2)' if (mask, sample) != best_combo else 'rgba(255,0,0,0.8)')
+
+    # Sampling methods to output
+    output_idx = len(label_list) - 1
+    for i, sample in enumerate(pipeline_metrics['sampling_methods']):
+        source.append(sampling_start + i)
+        target.append(output_idx)
+        value.append(1)
+        colors.append('rgba(0,0,255,0.2)' if sample != best_combo[1] else 'rgba(255,0,0,0.8)')
+
+    fig = go.Figure(data=[go.Sankey(
+        node=dict(
+            pad=15,
+            thickness=20,
+            line=dict(color="black", width=0.5),
+            label=label_list,
+            color="lightblue"
+        ),
+        link=dict(
+            source=source,
+            target=target,
+            value=value,
+            color=colors
+        )
+    )])
+    
+    fig.update_layout(
+        title_text=f"Watermarking Pipeline Flow<br>Best Combination: {best_combo[0]} + {best_combo[1]}",
+        font_size=12,
+        height=500
+    )
+    return fig