Modified for hallucination evaluation task

scripts/create_request_file.py

@@ -7,9 +7,10 @@ from datetime import datetime, timezone
 import click
 from colorama import Fore
 from huggingface_hub import HfApi, snapshot_download
+from util import QUEUE_REPO, EVAL_REQUESTS_PATH
 
-EVAL_REQUESTS_PATH = "eval-queue"
-QUEUE_REPO = "open-llm-leaderboard/requests"
+# EVAL_REQUESTS_PATH = "eval-queue"
+# QUEUE_REPO = "open-llm-leaderboard/requests"
 
 precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
 model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned")

src/backend/evaluate_model.py

@@ -0,0 +1,37 @@
+import pandas as pd
+
+from leaderboard.src.backend.model_operations import SummaryGenerator, EvaluationModel
+from envs import HEM_PATH, SOURCE_PATH
+from leaderboard.src.backend.util import load_dataframe, format_results
+
+class Evaluator:
+    def __init__(self, model, revision, precision, num_fewshot, batch_size, device, no_cache, limit, write_out=True, output_base_path='logs'):
+        self.model = model
+        self.revision = revision
+        self.precision = precision
+        self.num_fewshot = num_fewshot
+        self.batch_size = batch_size
+        self.device = device
+        self.no_cache = no_cache
+        self.limit = limit
+        self.write_out = write_out
+        self.output_base_path = output_base_path
+        self.summary_generator = SummaryGenerator(model, revision)
+        self.eval_model = EvaluationModel(HEM_PATH)
+    
+    def evaluate(self):
+        df = load_dataframe(SOURCE_PATH)
+        generated_summaries_df = self.summary_generator.generate_summaries(df)
+        
+        avg_summary_len = self.summary_generator.avg_length
+        answer_rate = self.summary_generator.answer_rate
+        
+        hallucination_scores = self.eval_model.evaluate_hallucination(generated_summaries_df)
+        
+        accuracy = self.eval_model.compute_accuracy
+        hallucination_rate = self.eval_model.hallucination_rate
+        
+        results = format_results(hallucination_scores, self.model, self.revision, self.precision, accuracy, hallucination_rate, answer_rate, avg_summary_len)
+        
+        return results
+    

src/backend/manage_requests.py

@@ -0,0 +1,122 @@
+import glob
+import json
+from dataclasses import dataclass
+from typing import Optional
+
+from huggingface_hub import HfApi, snapshot_download
+from src.envs import TOKEN
+
+@dataclass
+class EvalRequest:
+    model: str
+    private: bool
+    status: str
+    json_filepath: str
+    weight_type: str = "Original"
+    model_type: str = ""  # pretrained, finetuned, with RL
+    precision: str = ""  # float16, bfloat16
+    base_model: Optional[str] = None # for adapter models
+    revision: str = "main" # commit
+    submitted_time: Optional[str] = "2022-05-18T11:40:22.519222"  # random date just so that we can still order requests by date
+    model_type: Optional[str] = None
+    likes: Optional[int] = 0
+    params: Optional[int] = None
+    license: Optional[str] = ""
+    
+    def get_model_args(self):
+        model_args = f"pretrained={self.model},revision={self.revision}"
+
+        if self.precision in ["float16", "bfloat16"]:
+            model_args += f",dtype={self.precision}"
+        # Quantized models need some added config, the install of bits and bytes, etc
+        #elif self.precision == "8bit":
+        #    model_args += ",load_in_8bit=True"
+        #elif self.precision == "4bit":
+        #    model_args += ",load_in_4bit=True"
+        #elif self.precision == "GPTQ":
+            # A GPTQ model does not need dtype to be specified,
+            # it will be inferred from the config
+            pass
+        else:
+            raise Exception(f"Unknown precision {self.precision}.")
+        
+        return model_args
+
+
+def set_eval_request(api: HfApi, eval_request: EvalRequest, set_to_status: str, hf_repo: str, local_dir: str):
+    """Updates a given eval request with its new status on the hub (running, completed, failed, ...)"""
+    json_filepath = eval_request.json_filepath
+
+    with open(json_filepath) as fp:
+        data = json.load(fp)
+
+    data["status"] = set_to_status
+
+    with open(json_filepath, "w") as f:
+        f.write(json.dumps(data))
+
+    api.upload_file(
+        path_or_fileobj=json_filepath,
+        path_in_repo=json_filepath.replace(local_dir, ""),
+        repo_id=hf_repo,
+        repo_type="dataset",
+    )
+
+
+def get_eval_requests(job_status: list, local_dir: str, hf_repo: str) -> list[EvalRequest]:
+    """Get all pending evaluation requests and return a list in which private
+    models appearing first, followed by public models sorted by the number of
+    likes.
+
+    Returns:
+        `list[EvalRequest]`: a list of model info dicts.
+    """
+    snapshot_download(repo_id=hf_repo, revision="main", local_dir=local_dir, repo_type="dataset", max_workers=60)
+    json_files = glob.glob(f"{local_dir}/**/*.json", recursive=True)
+
+    eval_requests = []
+    for json_filepath in json_files:
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+        if data["status"] in job_status:
+            data["json_filepath"] = json_filepath
+            eval_request = EvalRequest(**data)
+            eval_requests.append(eval_request)
+
+    return eval_requests
+
+
+def check_completed_evals(
+    api: HfApi,
+    hf_repo: str,
+    local_dir: str,
+    checked_status: str,
+    completed_status: str,
+    failed_status: str,
+    hf_repo_results: str,
+    local_dir_results: str,
+):
+    """Checks if the currently running evals are completed, if yes, update their status on the hub."""
+    snapshot_download(repo_id=hf_repo_results, revision="main", local_dir=local_dir_results, repo_type="dataset", max_workers=60)
+
+    running_evals = get_eval_requests(checked_status, hf_repo=hf_repo, local_dir=local_dir)
+
+    for eval_request in running_evals:
+        model = eval_request.model
+        print("====================================")
+        print(f"Checking {model}")
+
+        output_path = model
+        output_file = f"{local_dir_results}/{output_path}/results*.json"
+        output_file_exists = len(glob.glob(output_file)) > 0
+
+        if output_file_exists:
+            print(
+                f"EXISTS output file exists for {model} setting it to {completed_status}"
+            )
+            set_eval_request(api, eval_request, completed_status, hf_repo, local_dir)
+        else:
+            print(
+                f"No result file found for {model} setting it to {failed_status}"
+            )
+            set_eval_request(api, eval_request, failed_status, hf_repo, local_dir)

src/backend/model_operations.py

@@ -0,0 +1,96 @@
+import numpy as np
+import pandas as pd
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from sentence_transformers import CrossEncoder
+
+from leaderboard.src.backend.util import generate_prompt
+
+def load_evaluation_model(model_path):
+    model = CrossEncoder(model_path)
+    model.save_pretrained('.checkpoints/{model_path}')
+    return model
+
+class SummaryGenerator:
+    def __init__(self, model_id, revision):
+        self.tokenizer = AutoTokenizer.from_pretrained(model_id, revision)
+        self.model = AutoModelForCausalLM.from_pretrained(model_id, revision)
+        self.summaries_df = pd.DataFrame()
+        self.revision = revision
+        self.avg_length = None
+        self.answer_rate = None
+
+    def generate_summaries(self, df):
+        source, summary, dataset = [], [], []
+
+        for index, row in df.iterrows():
+            _source = row['text']
+            _dataset = row['dataset']
+
+            prompt = generate_prompt(_source)
+            inputs = self.tokenizer(prompt, return_tensors='pt', max_length=1024, revision=self.revision)
+            try:
+                outputs = self.model.generate(**inputs, max_new_tokens=1024, do_sample=False, temperature=0.0, revision=self.revision)
+                response = self.tokenizer.decode(outputs[0], skip_special_tokens=True, revision=self.revision)
+            except Exception as e:
+                print(f"Error at index {index}: {e}")
+                response = ""
+                
+            summary.append(response)
+            source.append(_source)
+            dataset.append(_dataset)
+
+        self.summaries_df = pd.DataFrame(list(zip(source, summary, dataset)), columns=["source", "summary", "dataset"])
+        self._compute_avg_length()
+        self._compute_answer_rate()
+
+        return self.summaries_df
+
+    def _compute_avg_length(self):
+        total_words = 0
+        count = 0
+
+        for summary in self.summaries_df['summary']:
+            if summary != "":
+                words = summary.split()
+                total_words += len(words)
+                count += 1
+
+        self.avg_length = 0 if count == 0 else total_words / count
+
+    def _compute_answer_rate(self):
+        non_empty_count = sum(1 for summary in self.summaries_df['summary'] if summary != "")
+        total_rows = len(self.summaries_df)
+
+        self.answer_rate = 0 if total_rows == 0 else non_empty_count / total_rows
+
+class EvaluationModel:
+    def __init__(self, model_path):
+        self.model = load_evaluation_model(model_path)
+        self.scores = []
+        self.accuracy = None
+        self.hallucination_rate = None
+
+    def evaluate_hallucination(self, summaries_df):
+        # Convert to NumPy arrays for efficient processing
+        source_docs = np.array(summaries_df['source'])
+        generated_summaries = np.array(summaries_df['summary'])
+
+        scores = self.model.predict(source_docs, generated_summaries)
+        self.scores = scores
+        return self.scores
+
+    def compute_accuracy(self):
+        if not self.scores:
+            raise ValueError("Scores not calculated. Call evaluate_hallucination() first.")
+
+        # Use threshold of 0.5 to compute accuracy
+        num_above_threshold = sum(score >= 0.5 for score in self.scores)
+        num_total = len(self.scores)
+
+        if num_total == 0:
+            raise ValueError("No scores available to compute accuracy.")
+
+        self.accuracy = (num_above_threshold / num_total) * 100
+        self.hallucination_rate = 100 - self.accuracy
+
+        return self.accuracy

src/backend/run_eval_suite.py

@@ -0,0 +1,49 @@
+import json
+import os
+import logging
+from datetime import datetime
+
+# from lm_eval import tasks, evaluator, utils
+from evaluate_model import Evaluator
+
+from src.envs import RESULTS_REPO, API
+from src.backend.manage_requests import EvalRequest
+
+from util import load_dataframe, format_results
+
+logging.getLogger("openai").setLevel(logging.WARNING)
+
+def run_evaluation(eval_request: EvalRequest, num_fewshot, batch_size, device, local_dir: str, results_repo: str, no_cache=True, limit=None):
+    if limit:
+        print(
+            "WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
+        )
+
+    # task_names = utils.pattern_match(task_names, tasks.ALL_TASKS)
+
+    # print(f"Selected Tasks: {task_names}")
+    evaluator = Evaluator(eval_request.model, eval_request.revision, eval_request.precision, num_fewshot, batch_size, device, no_cache, limit, write_out=True, output_base_path='logs')
+    results = evaluator.evaluate()
+    
+    # results["config"]["model_dtype"] = eval_request.precision
+    # results["config"]["model_name"] = eval_request.model
+    # results["config"]["model_sha"] = eval_request.revision
+
+    dumped = json.dumps(results, indent=2)
+    print(dumped)
+
+    output_path = os.path.join(local_dir, *eval_request.model.split("/"), f"results_{datetime.now()}.json")
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    with open(output_path, "w") as f:
+        f.write(dumped)
+
+    print(evaluator.make_table(results))
+
+    API.upload_file(
+        path_or_fileobj=output_path,
+        path_in_repo=f"{eval_request.model}/results_{datetime.now()}.json",
+        repo_id=results_repo,
+        repo_type="dataset",
+    )
+
+    return results

src/backend/sort_queue.py

@@ -0,0 +1,28 @@
+import re
+from dataclasses import dataclass
+
+from huggingface_hub import HfApi
+
+from src.backend.manage_requests import EvalRequest
+
+
+@dataclass
+class ModelMetadata:
+    likes: int = 0
+    size: int = 15
+
+
+def sort_models_by_priority(api: HfApi, models: list[EvalRequest]) -> list[EvalRequest]:
+    private_models = [model for model in models if model.private]
+    public_models = [model for model in models if not model.private]
+
+    return sort_by_submit_date(private_models) + sort_by_submit_date(public_models)
+
+def sort_by_submit_date(eval_requests: list[EvalRequest]) -> list[EvalRequest]:
+    return sorted(eval_requests, key=lambda x: x.submitted_time, reverse=False)
+
+def sort_by_size(eval_requests: list[EvalRequest]) -> list[EvalRequest]:
+    return sorted(eval_requests, key=lambda x: x.params, reverse=False)
+
+def sort_by_likes(eval_requests: list[EvalRequest]) -> list[EvalRequest]:
+    return sorted(eval_requests, key=lambda x: x.likes, reverse=False)

src/backend/util.py

@@ -0,0 +1,32 @@
+import pandas as pd
+
+def load_dataframe(data_path):
+    df = pd.read_csv(data_path)
+    return df
+
+def generate_prompt(source_passage):
+    return f"""You are a chat bot answering questions using data. You must stick to the answers provided solely by the text in the passage provided. 
+    You are asked the question 'Provide a concise summary of the following passage, covering the core pieces of information described.'
+    Passage:
+    {source_passage}
+    """
+    
+def format_results(hallucination_scores, model_name, revision, precision, accuracy, hallucination_rate, answer_rate, avg_summary_len):
+    # Define the structure of the results (JSON)
+    results = {
+        "config": {
+            "model_dtype": precision, # Precision with which you ran the evaluation
+            "model_name": model_name, # Name of the model
+            "model_sha": revision # Hash of the model 
+        },
+        "results": {
+            "hallucination_eval": {
+                "HEM Scores": hallucination_scores,
+                "Accuracy": accuracy,
+                "Hallucination Rate": hallucination_rate,
+                "Answer Rate": answer_rate, 
+                "Average Summary Length": avg_summary_len,
+            }
+        }
+    }
+    return results

src/display/about.py

@@ -1,18 +1,18 @@
 from dataclasses import dataclass
 from enum import Enum
 
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Init: to update with your specific keys
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("task_name1", "metric_name", "First task")
-    task1 = Task("task_name2", "metric_name", "Second task")
+# @dataclass
+# class Task:
+#     benchmark: str
+#     metric: str
+#     col_name: str
+
+
+# # Init: to update with your specific keys
+# class Tasks(Enum):
+#     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+#     task0 = Task("task_name1", "metric_name", "First task")
+#     task1 = Task("task_name2", "metric_name", "Second task")
 
 
 # Your leaderboard name

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.display.about import Tasks
+# from src.display.about import Tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -26,10 +26,15 @@ auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Accuracy
+auto_eval_column_dict.append(["accuracy", ColumnContent, ColumnContent("Accuracy ⬆️", "number", True)])
+# Hallucination Rate
+auto_eval_column_dict.append(["hallucination_rate", ColumnContent, ColumnContent("Hallucination Rate ⬇️", "number", True)])
+# Answer Rate
+auto_eval_column_dict.append(["answer_rate", ColumnContent, ColumnContent("Answer Rate ⬆️", "number", True)])
+# Average Summary Length
+auto_eval_column_dict.append(["average_summary_length", ColumnContent, ColumnContent("Average Summary Length", "number", True)])
+
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
@@ -121,7 +126,7 @@ TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default a
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+BENCHMARK_COLS = ["Accuracy", "Hallucination Rate", "Answer Rate", "Average Summary Length"]
 
 NUMERIC_INTERVALS = {
     "?": pd.Interval(-1, 0, closed="right"),

src/envs.py

@@ -2,10 +2,10 @@ import os
 
 from huggingface_hub import HfApi
 
-# clone / pull the lmeh eval data
-TOKEN = os.environ.get("TOKEN", None)
+# replace this with our token 
+TOKEN = os.environ.get("HF_TOKEN", None)
 
-OWNER = "demo-leaderboard"
+OWNER = "vectara"
 REPO_ID = f"{OWNER}/leaderboard"
 QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
@@ -17,3 +17,6 @@ EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
 
 API = HfApi(token=TOKEN)
+
+SOURCE_PATH = "/datasets/leaderboard_summaries.csv"
+HEM_PATH = 'vectara/hallucination_evaluation_model'

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Precision, WeightType
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -65,17 +65,16 @@ class EvalResult:
                 architecture = ";".join(architectures)
 
         # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
+        hallucination_eval = data["results"].get("hallucination_eval", {})
+        
+        # Extract metrics from hallucination eval
+        results = {
+            "HEM Scores": hallucination_eval.get("HEM Scores", None),
+            "Accuracy": hallucination_eval.get("Accuracy", None),
+            "Hallucination Rate": hallucination_eval.get("Hallucination Rate", None),
+            "Answer Rate": hallucination_eval.get("Answer Rate", None),
+            "Average Summary Length": hallucination_eval.get("Average Summary Length", None),
+        }
 
         return self(
             eval_name=result_key,
@@ -107,7 +106,9 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        accuracy = self.results.get("Accuracy", None)
+        
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -118,15 +119,18 @@ class EvalResult:
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.dummy.name: self.full_model,
             AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            # AutoEvalColumn.average.name: average,
+            AutoEvalColumn.accuracy.name: accuracy,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+        # for task in Tasks:
+        #     data_dict[task.value.col_name] = self.results[task.value.benchmark]
+        data_dict["Hallucination Rate"] = self.results.get("Hallucination Rate", None)
+        data_dict["Answer Rate"] = self.results.get("Answer Rate", None)
+        data_dict["Average Summary Length"] = self.results.get("Average Summary Length", None)
 
         return data_dict
 

src/populate.py

@@ -13,7 +13,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df.sort_values(by=[AutoEvalColumn.accuracy.name], ascending=False)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced

src/submission/submit.py

@@ -94,6 +94,7 @@ def add_new_eval(
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
+
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
     out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"