..

app.py

@@ -237,88 +237,77 @@ def format_chat_history(message, chat_history):
         formatted_chat_history.append(f"User: {user_message}")
         formatted_chat_history.append(f"Assistant: {bot_message}")
     return formatted_chat_history
-    
-
-
-
-
-###############################################
-class RAGEvaluator:
-    def __init__(self):
-        self.gpt2_model, self.gpt2_tokenizer = self.load_gpt2_model()
-        self.bias_pipeline = pipeline("zero-shot-classification", model="Hate-speech-CNERG/dehatebert-mono-english")
-
-    def load_gpt2_model(self):
-        model = GPT2LMHeadModel.from_pretrained('gpt2')
-        tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
-        return model, tokenizer
-
-    def evaluate_bleu_rouge(self, candidates, references):
-        bleu_score = corpus_bleu(candidates, [references]).score
-        scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)
-        rouge_scores = [scorer.score(ref, cand) for ref, cand in zip(references, candidates)]
-        rouge1 = sum([score['rouge1'].fmeasure for score in rouge_scores]) / len(rouge_scores)
-        return bleu_score, rouge1
-
-    def evaluate_bert_score(self, candidates, references):
-        P, R, F1 = score(candidates, references, lang="en", model_type='bert-base-multilingual-cased')
-        return P.mean().item(), R.mean().item(), F1.mean().item()
-
-    def evaluate_perplexity(self, text):
-        encodings = self.gpt2_tokenizer(text, return_tensors='pt')
-        max_length = self.gpt2_model.config.n_positions
-        stride = 512
-        lls = []
-        for i in range(0, encodings.input_ids.size(1), stride):
-            begin_loc = max(i + stride - max_length, 0)
-            end_loc = min(i + stride, encodings.input_ids.size(1))
-            trg_len = end_loc - i
-            input_ids = encodings.input_ids[:, begin_loc:end_loc]
-            target_ids = input_ids.clone()
-            target_ids[:, :-trg_len] = -100
-            with torch.no_grad():
-                outputs = self.gpt2_model(input_ids, labels=target_ids)
-                log_likelihood = outputs[0] * trg_len
-            lls.append(log_likelihood)
-        ppl = torch.exp(torch.stack(lls).sum() / end_loc)
-        return ppl.item()
-
-    def evaluate_diversity(self, texts):
-        all_tokens = [tok for text in texts for tok in text.split()]
-        unique_bigrams = set(ngrams(all_tokens, 2))
-        diversity_score = len(unique_bigrams) / len(all_tokens) if all_tokens else 0
-        return diversity_score
-
-    def evaluate_racial_bias(self, text):
-        results = self.bias_pipeline([text], candidate_labels=["hate speech", "not hate speech"])
-        bias_score = results[0]['scores'][results[0]['labels'].index('hate speech')]
-        return bias_score
-
-    def evaluate_all(self, question, response, reference):
-        candidates = [response]
-        references = [reference]
-        bleu, rouge1 = self.evaluate_bleu_rouge(candidates, references)
-        bert_p, bert_r, bert_f1 = self.evaluate_bert_score(candidates, references)
-        perplexity = self.evaluate_perplexity(response)
-        diversity = self.evaluate_diversity(candidates)
-        racial_bias = self.evaluate_racial_bias(response)
-        return {
-            "BLEU": bleu,
-            "ROUGE-1": rouge1,
-            "BERT P": bert_p,
-            "BERT R": bert_r,
-            "BERT F1": bert_f1,
-            "Perplexity": perplexity,
-            "Diversity": diversity,
-            "Racial Bias": racial_bias
-        }
-
-###################################
 
-evaluator = RAGEvaluator()
-
-
-#################################
+#----------------------------------------------------------------------------------
+def load_gpt2_model():
+    model = GPT2LMHeadModel.from_pretrained('gpt2')
+    tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
+    return model, tokenizer
+
+gpt2_model, gpt2_tokenizer = load_gpt2_model()
+bias_pipeline = pipeline("zero-shot-classification", model="Hate-speech-CNERG/dehatebert-mono-english")
+
+def evaluate_bleu_rouge(candidates, references):
+    bleu_score = corpus_bleu(candidates, [references]).score
+    scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)
+    rouge_scores = [scorer.score(ref, cand) for ref, cand in zip(references, candidates)]
+    rouge1 = sum([score['rouge1'].fmeasure for score in rouge_scores]) / len(rouge_scores)
+    return bleu_score, rouge1
+
+def evaluate_bert_score(candidates, references):
+    P, R, F1 = score(candidates, references, lang="en", model_type='bert-base-multilingual-cased')
+    return P.mean().item(), R.mean().item(), F1.mean().item()
+
+def evaluate_perplexity(text, model, tokenizer):
+    encodings = tokenizer(text, return_tensors='pt')
+    max_length = model.config.n_positions
+    stride = 512
+    lls = []
+    for i in range(0, encodings.input_ids.size(1), stride):
+        begin_loc = max(i + stride - max_length, 0)
+        end_loc = min(i + stride, encodings.input_ids.size(1))
+        trg_len = end_loc - i
+        input_ids = encodings.input_ids[:, begin_loc:end_loc]
+        target_ids = input_ids.clone()
+        target_ids[:, :-trg_len] = -100
+        with torch.no_grad():
+            outputs = model(input_ids, labels=target_ids)
+            log_likelihood = outputs[0] * trg_len
+        lls.append(log_likelihood)
+    ppl = torch.exp(torch.stack(lls).sum() / end_loc)
+    return ppl.item()
+
+def evaluate_diversity(texts):
+    all_tokens = [tok for text in texts for tok in text.split()]
+    unique_bigrams = set(ngrams(all_tokens, 2))
+    diversity_score = len(unique_bigrams) / len(all_tokens) if all_tokens else 0
+    return diversity_score
+
+def evaluate_racial_bias(text, pipeline):
+    results = pipeline([text], candidate_labels=["hate speech", "not hate speech"])
+    bias_score = results[0]['scores'][results[0]['labels'].index('hate speech')]
+    return bias_score
+
+def evaluate_all(question, response, reference, gpt2_model, gpt2_tokenizer, bias_pipeline):
+    candidates = [response]
+    references = [reference]
+    bleu, rouge1 = evaluate_bleu_rouge(candidates, references)
+    bert_p, bert_r, bert_f1 = evaluate_bert_score(candidates, references)
+    perplexity = evaluate_perplexity(response, gpt2_model, gpt2_tokenizer)
+    diversity = evaluate_diversity(candidates)
+    racial_bias = evaluate_racial_bias(response, bias_pipeline)
+    return {
+        "BLEU": bleu,
+        "ROUGE-1": rouge1,
+        "BERT P": bert_p,
+        "BERT R": bert_r,
+        "BERT F1": bert_f1,
+        "Perplexity": perplexity,
+        "Diversity": diversity,
+        "Racial Bias": racial_bias
+    }
+
+#---------------------------------------------------------------------------------
 
 def display_metrics(metrics):
     result = ""
@@ -339,8 +328,14 @@ def display_metrics(metrics):
         elif k == 'Racial Bias':
             result += f"Racial Bias score indicates the presence of biased language in the generated output. Higher scores indicate more bias. Score obtained: {v}\n\n"
     return result
+#---------------------------------------------------------------------------------------------------------------------------------------------------
+
+
+
+
+
 
-def conversation(qa_chain, message, history, evaluator):
+def conversation(qa_chain, message, history, gpt2_model, gpt2_tokenizer, bias_pipeline):
     formatted_chat_history = format_chat_history(message, history)
     question_by_user = message
     
@@ -363,7 +358,7 @@ def conversation(qa_chain, message, history, evaluator):
     new_history = history + [(message, response_answer)]
     
     # Evaluate the metrics
-    metrics = evaluator.evaluate_all(question_by_user, answer_of_question, context)
+    metrics = evaluate_all(question_by_user, answer_of_question, context)
     evaluation_metrics = display_metrics(metrics)
 
     return (qa_chain, gr.update(value=""), new_history, response_source1, response_source1_page,
@@ -470,12 +465,12 @@ def demo():
 
         # Chatbot events
         msg.submit(conversation, \
-            inputs=[qa_chain, msg, chatbot,evaluator], \
+            inputs=[qa_chain, msg, chatbot], \
             outputs=[qa_chain, msg, chatbot, doc_source1, source1_page, doc_source2, source2_page, doc_source3, source3_page,metrics_output], \
             queue=False)
 
         submit_btn.click(conversation, 
-                         inputs=[qa_chain, msg, history,evaluator], 
+                         inputs=[qa_chain, msg, history], 
                          outputs=[qa_chain, chatbot, history, doc_source1, source1_page, doc_source2, source2_page, doc_source3, source3_page, metrics_output])
 
         clear_btn.click(lambda: [None, "", 0, "", 0, "", 0],