update mt_bench

README.md

@@ -1,11 +1,5 @@
----
-title: Tlem
-emoji: 🏆
-colorFrom: yellow
-colorTo: yellow
-sdk: static
-pinned: false
-license: apache-2.0
----
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Transparent LLMs Evaluation Metrics
+
+> LLMs belong to *tout le monde*

tasks.py

@@ -14,16 +14,42 @@ from evaluate import load
 from collections import defaultdict
 import sys
 
+
 # if sys.version_info >= (3, 9):
 #     from functools import cache
 # else:
 #     from functools import lru_cache as cache
 
+
 disable_progress_bar()
 
 
-def fake_pipeline(prompts: Iterable[str]) -> list[str]:
-    return [prompt for prompt in tqdm(prompts)]
+def mt_bench_prompt(example):
+    judge_prompt = "You are ChatGPT, a large language model trained by OpenAI. Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. The  Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response."
+    judge_prompt = "You are ChatGPT, a large language model trained by OpenAI. Your task is to act as an impartial judge and evaluate the quality of the responses provided by an 'assistant' role in the displayed conversation. Your evaluation should focus on the helpfulness, relevance, accuracy, depth, creativity, language fluency, clarity, and level of detail in the assistant's responses. Please note that the evaluation should not consider the user's questions or the overall conversation, but solely the quality of the assistant's replies."
+    multi_prompt = "You evaluation should focus on the assistant's answer to the second user question."
+    ref_prompt = "In the conversation, you will encounter system messages labeled 'Reference Answer' followed by the assistant's response. Your task is to evaluate the quality of the assistant's response by comparing it with the reference answer."
+    json_prompt = 'You must rate the response on a scale of 1 to 10 in JSON format, for example: {"rating": 5}.'
+    prompt_list = [judge_prompt]
+    conversations = example["conversation"]
+    if example["turn"] == 2:
+        prompt_list.append(multi_prompt)
+
+    if example["reference"] is not None:
+        conversations = []
+        quesiotns = filter(lambda e: e["role"] == "user", example["conversation"])
+        answers = filter(lambda e: e["role"] == "assistant", example["conversation"])
+        for q, a, r in zip(quesiotns, answers, example["reference"]):
+            conversations.append(q)
+            conversations.append(
+                {"role": "system", "content": "Reference Answer: " + r}
+            )
+            conversations.append(a)
+        prompt_list.append(ref_prompt)
+    prompt_list.append(json_prompt)
+
+    messages = [{"role": "system", "content": " ".join(prompt_list)}] + conversations
+    return messages
 
 
 @dataclass
@@ -33,7 +59,7 @@ class Task:
     # metrics: list[str] = field(default_factory=list)
     metric_name: str | tuple[str, str] = ("sustech/tlem", "gsm8k")
     input_column: str = "question"
-    label_column: str = "answer"
+    label_column: str = ""
     prompt: Optional[Callable | str] = None
     few_shot: int = 0
     few_shot_from: Optional[str] = None
@@ -54,6 +80,7 @@ class Task:
                     input_column=example[self.input_column]
                 )
             }
+        self.label_column = self.label_column or self.input_column
 
     @cached_property
     def samples(self):
@@ -78,6 +105,7 @@ class Task:
                         self.few_shot_from = name
                         break
 
+            assert self.few_shot_from != self.split
             shots = ds[self.few_shot_from].select(range(self.few_shot))
             if self.prompt is not None:
                 shots = shots.map(self.prompt)
@@ -126,6 +154,8 @@ class Task:
             result = self.metric.compute(
                 responses=outputs, references=self.dataset[self.label_column]
             )
+        finally:
+            result = outputs
         # if log:
         #     name = name or pipeline.__name__
         #     self.results[name] = result
@@ -431,7 +461,6 @@ class MMLU:
             "psychology",
         ],
         "other": ["other", "business", "health"],
-        "Test": ["culture"],
     }
 
     @classmethod

tlem.py

@@ -69,15 +69,9 @@ class ReasoningMetric(evaluate.Metric):
         return results
 
 
-gsm8k = Task(
-    dataset_name=("gsm8k", "main"),
-    metric_name=("sustech/tlem", "gsm8k"),
-    input_column="question",
-    label_column="answer",
-)
-
-
 class Suite(EvaluationSuite):
+    task_class = Task
+
     def run(
         self,
         model_or_pipeline: Any,
@@ -115,12 +109,21 @@ class Suite(EvaluationSuite):
             case _ if name.startswith("cmmlu"):
                 suite = CMMLU.suite(chat=chat)
             case "gsm8k":
-                suite = [gsm8k]
-        match name:
-            case _ if "test" in name:
-                suite = suite["Test"]
-
-        self.suite = suite
+                suite = Task(
+                    dataset_name=("gsm8k", "main"),
+                    metric_name=("sustech/tlem", "gsm8k"),
+                    input_column="question",
+                    label_column="answer",
+                )
+            case "mt_bench":
+                suite = Task(
+                    dataset_name="SUSTech/mt_bench_judge",
+                    split="train",
+                    prompt=mt_bench_prompt
+                    # metric_name=("sustech/tlem", "gsm8k"),
+                )
+
+        self.suite = [suite] if isinstance(suite, Task) else suite
 
     def __init__(self, name="tlem"):
         super().__init__(name)