update

cli/halueval-cli.py

@@ -46,7 +46,7 @@ def main():
 
     for task in TASKS_HARNESS:
         print(f"Selected Tasks: [{task}]")
-        results = evaluator.simple_evaluate(model="hf", model_args=eval_request.get_model_args(), tasks=[task.benchmark], num_fewshot=1,
+        results = evaluator.simple_evaluate(model="hf", model_args=eval_request.get_model_args(), tasks=[task.benchmark], num_fewshot=4,
                                             batch_size=1, device="mps", use_cache=None, limit=10, write_out=True)
         print('AAA', results["results"])
 

requirements.txt

@@ -25,3 +25,4 @@ sacrebleu
 cchardet
 rouge_score
 bert-score
+evaluate

src/backend/tasks/xsum/task.py

@@ -61,6 +61,7 @@ class XSum(Task):
         super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
         self.factkb_tokenizer = None
         self.factkb_model = None
+        self.bert_score = None
 
     def maybe_init_factkb(self):
         if self.factkb_tokenizer is None or self.factkb_model is None:
@@ -68,6 +69,11 @@ class XSum(Task):
             self.factkb_tokenizer = AutoTokenizer.from_pretrained("roberta-base", padding="max_length", truncation=True)
             self.factkb_model = AutoModelForSequenceClassification.from_pretrained("bunsenfeng/FactKB", num_labels=2, device_map="auto")
 
+    def maybe_init_bertscore(self):
+        if self.bert_score is None:
+            from evaluate import load
+            self.bert_score = load("bertscore")
+
     def has_training_docs(self):
         return True
 
@@ -126,6 +132,8 @@ class XSum(Task):
         completion = results[0]
 
         document = doc["document"]
+        gold_summary = doc["summary"]
+
         true_refs = [doc["summary"]]
         all_refs = true_refs
 
@@ -144,11 +152,17 @@ class XSum(Task):
         factkb_logits = self.factkb_model(**factkb_tokens).logits
         factkb_res = torch.softmax(factkb_logits, dim=1)
 
+        self.maybe_init_factkb()
+        bert_score_res = self.bert_score.compute(predictions=[completion], references=[gold_summary], lang="en")
+
         res = {
             "rouge1": rouge1_scores[0],
             "rouge2": rouge2_scores[0],
             "rougeL": rougeL_scores[0],
-            "factKB": float(factkb_res[0][1])
+            "factKB": float(factkb_res[0][1]),
+            "bertscore_precision": float(bert_score_res["precision"][0]),
+            "bertscore_recall": float(bert_score_res["recall"][0]),
+            "bertscore_f1": float(bert_score_res["f1"][0]),
         }
 
         # breakpoint()