feat: add bbh/mmlu parser

llmdataparser/bbh_parser.py

@@ -0,0 +1,110 @@
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+from llmdataparser.base_parser import HuggingFaceDatasetParser, HuggingFaceParseEntry
+from llmdataparser.prompts import BBH_SYSTEM_PROMPT  # You'll need to create this
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class BBHParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for BBH (Big Bench Hard), with fields specific to this dataset."""
+
+    @classmethod
+    def create(
+        cls,
+        prompt: str,
+        answer: str,
+        raw_question: str,
+        raw_answer: str,
+        task_name: str,
+    ) -> "BBHParseEntry":
+        return cls(
+            prompt=prompt,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            task_name=task_name,
+        )
+
+
+class BBHDatasetParser(HuggingFaceDatasetParser[BBHParseEntry]):
+    """Parser for the Big Bench Hard dataset."""
+
+    _data_source: ClassVar[str] = "lukaemon/bbh"
+    _task_names: ClassVar[list[str]] = [
+        "boolean_expressions",
+        "causal_judgement",
+        "date_understanding",
+        "disambiguation_qa",
+        "dyck_languages",
+        "formal_fallacies",
+        "geometric_shapes",
+        "hyperbaton",
+        "logical_deduction_five_objects",
+        "logical_deduction_seven_objects",
+        "logical_deduction_three_objects",
+        "movie_recommendation",
+        "multistep_arithmetic_two",
+        "navigate",
+        "object_counting",
+        "penguins_in_a_table",
+        "reasoning_about_colored_objects",
+        "ruin_names",
+        "salient_translation_error_detection",
+        "snarks",
+        "sports_understanding",
+        "temporal_sequences",
+        "tracking_shuffled_objects_five_objects",
+        "tracking_shuffled_objects_seven_objects",
+        "tracking_shuffled_objects_three_objects",
+        "web_of_lies",
+        "word_sorting",
+    ]
+    _default_task: ClassVar[str] = "reasoning_about_colored_objects"
+    _default_system_prompt: ClassVar[str] = BBH_SYSTEM_PROMPT
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> BBHParseEntry:
+        """Process a single BBH entry."""
+        raw_question = row["input"]
+        raw_answer = row["target"]
+
+        # Remove parentheses from the answer
+        clean_answer = raw_answer.strip("()")
+
+        # Combine system prompt with the question
+        prompt = f"{self._system_prompt}\n\n{raw_question}"
+
+        # Use task_name if provided, otherwise use default
+        task = task_name or self._get_current_task(row)
+
+        return BBHParseEntry.create(
+            prompt=prompt,
+            answer=clean_answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            task_name=task,
+        )
+
+
+if __name__ == "__main__":
+    # Example usage
+    parser = BBHDatasetParser()
+
+    # Load the dataset with a specific task
+    parser.load(task_name="reasoning_about_colored_objects")
+
+    # Parse all splits
+    parser.parse()
+
+    # Get parsed data
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task: {example.task_name}")
+        print(f"Question: {example.raw_question}")
+        print(f"Answer: {example.answer}")

llmdataparser/mmlu_parser.py

@@ -1,81 +1,422 @@
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, Final
 
-from llmdataparser.base_parser import HuggingFaceDatasetParser, ParseEntry
-from llmdataparser.prompts import MMLU_SYSTEM_PROMPT
+from llmdataparser.base_parser import HuggingFaceDatasetParser, HuggingFaceParseEntry
+from llmdataparser.prompts import MMLU_PRO_SYSTEM_PROMPT, MMLU_SYSTEM_PROMPT
 
+MMLU_VALID_ANSWERS: Final[set[str]] = {"A", "B", "C", "D"}
+MMLU_PRO_VALID_ANSWERS: Final[set[str]] = {
+    "A",
+    "B",
+    "C",
+    "D",
+    "E",
+    "F",
+    "G",
+    "H",
+    "I",
+    "J",
+}
+MMLU_VALID_ANSWER_STR: Final[str] = ", ".join(sorted(MMLU_VALID_ANSWERS))
+MMLU_PRO_VALID_ANSWER_STR: Final[str] = ", ".join(sorted(MMLU_PRO_VALID_ANSWERS))
 
-@dataclass(frozen=True)
-class MMLUParseEntry(ParseEntry):
-    """
-    Custom entry class for MMLU, with fields specific to this dataset parser.
-    """
 
-    prompt: str
-    answer_letter: str
+@dataclass(frozen=True, kw_only=True, slots=True)
+class MMLUParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for MMLU, with fields specific to this dataset parser."""
+
+    raw_choices: list[str]
+    task_name: str
 
     @classmethod
-    def create(cls, prompt: str, answer_letter: str) -> "MMLUParseEntry":
-        if answer_letter not in {"A", "B", "C", "D"}:
+    def create(
+        cls,
+        prompt: str,
+        answer: str,
+        raw_question: str,
+        raw_choices: list[str],
+        raw_answer: str,
+        task_name: str,
+    ) -> "MMLUParseEntry":
+        if answer not in MMLU_VALID_ANSWERS:
             raise ValueError(
-                f"Invalid answer_letter '{answer_letter}'; must be one of 'A', 'B', 'C', 'D'."
+                f"Invalid answer_letter '{answer}'; must be one of {MMLU_VALID_ANSWER_STR}"
             )
-        return cls(prompt=prompt, answer_letter=answer_letter)
+        if not task_name:
+            raise ValueError("Task name cannot be empty")
+        return cls(
+            prompt=prompt,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            raw_choices=raw_choices,
+            task_name=task_name,
+        )
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class MMLUProParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for MMLU, with fields specific to this dataset parser."""
+
+    raw_choices: list[str]
+    task_name: str
+
+    @classmethod
+    def create(
+        cls,
+        prompt: str,
+        answer: str,
+        raw_question: str,
+        raw_choices: list[str],
+        raw_answer: str,
+        task_name: str,
+    ) -> "MMLUProParseEntry":
+        if answer not in MMLU_PRO_VALID_ANSWERS:
+            raise ValueError(
+                f"Invalid answer_letter '{answer}'; must be one of {MMLU_PRO_VALID_ANSWER_STR}"
+            )
+        if not task_name:
+            raise ValueError("Task name cannot be empty")
+        return cls(
+            prompt=prompt,
+            answer=answer,
+            raw_question=raw_question,
+            raw_choices=raw_choices,
+            raw_answer=raw_answer,
+            task_name=task_name,
+        )
 
 
 class MMLUDatasetParser(HuggingFaceDatasetParser[MMLUParseEntry]):
+    """Base class for MMLU dataset parsers with common functionality."""
+
+    _default_system_prompt = MMLU_SYSTEM_PROMPT
+
+    def _get_task_from_entry(self, data_entry: dict[str, Any]) -> str:
+        """Get the task name from the data entry or default task name."""
+        task_name = data_entry.get("subject")
+        return task_name if task_name else (self._current_task or self._default_task)
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> MMLUParseEntry:
+        """
+        Generate a prompt and expected answer from the given row.
+
+        Args:
+            row: A data point to be formatted.
+            task_name: Optional task name for the entry.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            MMLUParseEntry: The formatted entry object.
+        """
+        task = task_name or self._get_current_task(row)
+        # Ensure task is not None
+        final_task = task or self._default_task
+
+        choices = "\n".join(
+            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(row["choices"])
+        )
+        raw_question = row["question"]
+        raw_choices = row["choices"]
+        raw_answer = str(row["answer"])  # Ensure raw_answer is a string
+
+        prompt = f"{self._system_prompt}\nQuestion: {raw_question}\n{choices}\nAnswer:"
+        answer_letter = chr(65 + int(raw_answer))  # Convert index to 'A', 'B', 'C', 'D'
+
+        return MMLUParseEntry.create(
+            prompt=prompt,
+            answer=answer_letter,
+            raw_question=raw_question,
+            raw_choices=raw_choices,
+            raw_answer=raw_answer,
+            task_name=final_task,
+        )
+
+
+class BaseMMLUDatasetParser(MMLUDatasetParser):
+    """Parser for the original MMLU dataset."""
+
     _data_source = "cais/mmlu"
+    _default_task = "all"
+    _task_names = [
+        "abstract_algebra",
+        "anatomy",
+        "astronomy",
+        "business_ethics",
+        "clinical_knowledge",
+        "college_biology",
+        "college_chemistry",
+        "college_computer_science",
+        "college_mathematics",
+        "college_medicine",
+        "college_physics",
+        "computer_security",
+        "conceptual_physics",
+        "econometrics",
+        "electrical_engineering",
+        "elementary_mathematics",
+        "formal_logic",
+        "global_facts",
+        "high_school_biology",
+        "high_school_chemistry",
+        "high_school_computer_science",
+        "high_school_european_history",
+        "high_school_geography",
+        "high_school_government_and_politics",
+        "high_school_macroeconomics",
+        "high_school_mathematics",
+        "high_school_microeconomics",
+        "high_school_physics",
+        "high_school_psychology",
+        "high_school_statistics",
+        "high_school_us_history",
+        "high_school_world_history",
+        "human_aging",
+        "human_sexuality",
+        "international_law",
+        "jurisprudence",
+        "logical_fallacies",
+        "machine_learning",
+        "management",
+        "marketing",
+        "medical_genetics",
+        "miscellaneous",
+        "moral_disputes",
+        "moral_scenarios",
+        "nutrition",
+        "philosophy",
+        "prehistory",
+        "professional_accounting",
+        "professional_law",
+        "professional_medicine",
+        "professional_psychology",
+        "public_relations",
+        "security_studies",
+        "sociology",
+        "us_foreign_policy",
+        "virology",
+        "world_religions",
+    ]
+
+
+class MMLUReduxDatasetParser(MMLUDatasetParser):
+    """Parser for the MMLU Redux dataset."""
+
+    _data_source = "edinburgh-dawg/mmlu-redux"
+    _default_task = "anatomy"
+    _task_names = [
+        "anatomy",
+        "astronomy",
+        "business_ethics",
+        "clinical_knowledge",
+        "college_chemistry",
+        "college_computer_science",
+        "college_mathematics",
+        "college_medicine",
+        "college_physics",
+        "conceptual_physics",
+        "econometrics",
+        "electrical_engineering",
+        "formal_logic",
+        "global_facts",
+        "high_school_chemistry",
+        "high_school_geography",
+        "high_school_macroeconomics",
+        "high_school_mathematics",
+        "high_school_physics",
+        "high_school_statistics",
+        "high_school_us_history",
+        "human_aging",
+        "logical_fallacies",
+        "machine_learning",
+        "miscellaneous",
+        "philosophy",
+        "professional_accounting",
+        "professional_law",
+        "public_relations",
+        "virology",
+    ]
+
+
+class TMMLUPlusDatasetParser(MMLUDatasetParser):
+    """Parser for the TMMLU+ dataset."""
+
+    _data_source = "ikala/tmmluplus"
+    _default_task = "taiwanese_hokkien"
+    _task_names = [
+        "engineering_math",
+        "dentistry",
+        "traditional_chinese_medicine_clinical_medicine",
+        "clinical_psychology",
+        "technical",
+        "culinary_skills",
+        "mechanical",
+        "logic_reasoning",
+        "real_estate",
+        "general_principles_of_law",
+        "finance_banking",
+        "anti_money_laundering",
+        "ttqav2",
+        "marketing_management",
+        "business_management",
+        "organic_chemistry",
+        "advance_chemistry",
+        "physics",
+        "secondary_physics",
+        "human_behavior",
+        "national_protection",
+        "jce_humanities",
+        "politic_science",
+        "agriculture",
+        "official_document_management",
+        "financial_analysis",
+        "pharmacy",
+        "educational_psychology",
+        "statistics_and_machine_learning",
+        "management_accounting",
+        "introduction_to_law",
+        "computer_science",
+        "veterinary_pathology",
+        "accounting",
+        "fire_science",
+        "optometry",
+        "insurance_studies",
+        "pharmacology",
+        "taxation",
+        "trust_practice",
+        "geography_of_taiwan",
+        "physical_education",
+        "auditing",
+        "administrative_law",
+        "education_(profession_level)",
+        "economics",
+        "veterinary_pharmacology",
+        "nautical_science",
+        "occupational_therapy_for_psychological_disorders",
+        "basic_medical_science",
+        "macroeconomics",
+        "trade",
+        "chinese_language_and_literature",
+        "tve_design",
+        "junior_science_exam",
+        "junior_math_exam",
+        "junior_chinese_exam",
+        "junior_social_studies",
+        "tve_mathematics",
+        "tve_chinese_language",
+        "tve_natural_sciences",
+        "junior_chemistry",
+        "music",
+        "education",
+        "three_principles_of_people",
+        "taiwanese_hokkien",
+    ]
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> MMLUParseEntry:
+        """Process a single TMMLU+ entry."""
+        # Extract choices in order
+        raw_choices = [row["A"], row["B"], row["C"], row["D"]]
+        choices = "\n".join(
+            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(raw_choices)
+        )
+        raw_question = row["question"]
+        raw_answer = row["answer"]
+
+        prompt = f"{self._system_prompt}\nQuestion: {raw_question}\n{choices}\nAnswer:"
+        task = task_name or self._get_current_task(row)
 
-    def __init__(self, system_prompt: str = MMLU_SYSTEM_PROMPT):
-        super().__init__()  # Properly initialize the base class
-        self.parsed_data: list[MMLUParseEntry] = []
-        self.task_names: list[str] = []
-        self.subject_list: set[str] = set()
-        self.system_prompt: str = system_prompt
-        super().__init__()
-
-    def parse(self, split_names: str | list[str] | None = None, **kwargs: Any) -> None:
-        self.parsed_data.clear()
-        if self.raw_data is None:
-            raise ValueError("No data loaded. Please load the dataset first.")
-
-        if split_names is None:
-            split_names = self.task_names
-        elif isinstance(split_names, str):
-            split_names = [split_names]
-
-        for split_name in split_names:
-            if split_name not in self.task_names:
-                raise ValueError(f"Task '{split_name}' not found in the dataset.")
-
-            dataset_split = self.raw_data[split_name]
-            for index, entry in enumerate(dataset_split, start=1):
-                data_entry = self.process_entry(entry, **kwargs)
-                self._parsed_data.append(data_entry)
-                self.subject_list.add(entry.get("subject", "Unknown"))
-            print(f"Parsed {index} data points from task '{split_name}'.")
-
-        print(
-            f"Number of subjects: {len(self.subject_list)}. "
-            "For more details, please check the `self.subject_list` attribute."
+        return MMLUParseEntry.create(
+            prompt, raw_answer, raw_question, raw_choices, raw_answer, task
         )
 
-    def process_entry(self, row: dict[str, Any], **kwargs) -> MMLUParseEntry:
+
+class MMLUProDatasetParser(HuggingFaceDatasetParser[MMLUProParseEntry]):
+    """Parser for the MMLU Pro dataset."""
+
+    _data_source = "TIGER-Lab/MMLU-Pro"
+    _default_task = "default"
+    _task_names = [
+        "math",
+        "physics",
+        "chemistry",
+        "law",
+        "engineering",
+        "other",
+        "economics",
+        "health",
+        "psychology",
+        "business",
+        "biology",
+        "philosophy",
+        "computer_science",
+        "history",
+    ]
+    _default_system_prompt = MMLU_PRO_SYSTEM_PROMPT
+
+    def _get_task_from_entry(self, data_entry: dict[str, Any]) -> str:
+        """Get the task name from the data entry or default task name."""
+        if data_entry is not None:
+            task_name = data_entry.get("category")
+            if task_name:
+                return task_name
+        return self._current_task or self._default_task
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> MMLUProParseEntry:
         """
         Generate a prompt and expected answer from the given row.
 
         Args:
-            row (dict[str, Any]): A data point to be formatted.
+            row (dict[str, Any]): A data point to be formatted with MMLU Pro specific structure
+                containing 'question', 'options', 'answer', and 'answer_index' keys.
 
         Returns:
             MMLUParseEntry: The formatted entry object.
         """
+        task = task_name or self._get_current_task(row)
+        # Ensure task is not None
+        final_task = task or self._default_task
+
+        # Extract choices in order
+        raw_choices = row["options"]
         choices = "\n".join(
-            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(row["choices"])
+            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(raw_choices)
         )
-        prompt = (
-            f"{self.system_prompt}\nQuestion: {row['question']}\n{choices}\nAnswer:"
+        raw_question = row["question"]
+        raw_answer = row["answer"]
+        answer_index = row["answer_index"]
+
+        prompt = f"{self._system_prompt}\nQuestion: {raw_question}\n{choices}\nAnswer:"
+        answer_letter = chr(
+            65 + answer_index
+        )  # Convert index to 'A', 'B', 'C', 'D', etc.
+
+        return MMLUProParseEntry.create(
+            prompt, answer_letter, raw_question, raw_choices, raw_answer, final_task
         )
-        answer_letter = chr(65 + row["answer"])  # Convert index to 'A', 'B', 'C', 'D'
 
-        return MMLUParseEntry.create(prompt, answer_letter)
+
+if __name__ == "__main__":
+    # Example usage of MMLU Pro parser
+    parser = MMLUProDatasetParser()
+    parser.load()
+    parser.parse()
+
+    # Get parsed data with correct type
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task: {example.task_name}")
+        print(f"Question: {example.raw_question}")
+        print("Choices:")
+        for i, choice in enumerate(example.raw_choices):
+            print(f"{chr(65 + i)}. {choice}")
+        print(f"Correct Answer: {example.answer}")

llmdataparser/prompts.py

@@ -3,10 +3,121 @@ from typing import Final
 
 MMLU_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
     """\
-    You are an expert assistant for answering questions in a multiple-choice format. Each question has four answer options (A, B, C, D). Your task is to analyze each question and select the most accurate answer.
+    You are a highly knowledgeable expert tasked with answering multiple-choice questions across various academic and professional fields. Each question has four options (A, B, C, D). Your goal is to select the single most accurate answer based on factual knowledge.
 
     Instructions:
-    1. Answer Selection: Review the question and choose the best option.
-    2. Response Format: Reply with only the letter (A, B, C, or D) of your chosen answer, without additional explanation.
+    1. Carefully analyze the question and all answer options
+    2. Consider only verified, factual information
+    3. Select the most precise and accurate option
+    4. Respond with ONLY the letter (A, B, C, or D) - no explanations or additional text
+"""
+)
+
+MMLU_PRO_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    You are a highly knowledgeable expert tasked with answering multiple-choice questions across various academic and professional fields. Each question has ten options (A through J). Your goal is to select the single most accurate answer based on factual knowledge.
+
+    Instructions:
+    1. Carefully analyze the question and all answer options
+    2. Consider only verified, factual information
+    3. Select the most precise and accurate option
+    4. Respond with ONLY the letter (A through J) - no explanations or additional text
+"""
+)
+
+GSM8K_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    You are an expert mathematics tutor. Your task is to solve math word problems by breaking them down into clear, logical steps.
+
+    Instructions:
+    1. Read the problem carefully
+    2. Show your step-by-step reasoning
+    3. Ensure each step is clear and mathematically sound
+    4. End with the final numerical answer
+    5. Format your response as:
+       Let's solve this step by step:
+       1) [First step]
+       2) [Second step]
+       ...
+       Therefore, the answer is [number]
+"""
+)
+
+
+HUMANEVAL_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    You are an expert Python programmer tasked with implementing Python functions. Your goal is to write clean, efficient, and correct code that meets the specifications.
+
+    Instructions:
+    1. Read the function signature and docstring carefully
+    2. Implement only the function body, not the signature or docstring
+    3. Follow Python best practices and PEP 8 style guidelines
+    4. Write clear, readable code with appropriate variable names
+    5. Handle edge cases and input validation where necessary
+    6. Use type hints and ensure type safety
+    7. Optimize for both readability and performance
+    8. Add comments for complex logic or non-obvious implementations
+    9. Include appropriate error handling with specific exception types
+    10. Consider writing code that would be easy to test
+    11. Return only the implementation code, no additional text
+
+    Example of good implementation:
+    ```python
+    # Handle edge case of empty input
+    if not numbers:
+        raise ValueError("Input list cannot be empty")
+
+    # Use descriptive variable names and type hints
+    result: list[int] = sorted(numbers)
+    return result[len(result) // 2]  # Return median value
+    ```
+"""
+)
+
+MGSM_SYSTEM_PROMPT = textwrap.dedent(
+    """\
+    You are an expert mathematics tutor who can explain solutions in multiple languages. Your task is to solve math word problems by breaking them down into clear, logical steps.
+
+    Instructions:
+    1. Read the problem carefully
+    2. Show your step-by-step reasoning
+    3. Ensure each step is clear and mathematically sound
+    4. Use appropriate number formatting for the target language (e.g., decimal points vs. commas)
+    5. End with the final numerical answer
+    6. Format your response as:
+       Let's solve this step by step:
+       1) [First step]
+       2) [Second step]
+       ...
+       Therefore, the answer is [number]
+"""
+)
+
+
+IFEVAL_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    You are a precise instruction follower. Your task is to generate responses that exactly match given requirements and constraints.
+
+    Instructions:
+    1. Read all requirements carefully
+    2. Follow formatting rules exactly
+    3. Meet all length requirements
+    4. Include all required elements
+    5. Avoid forbidden elements
+    6. Provide ONLY the requested output
+"""
+)
+
+BBH_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    You are a highly intelligent expert tasked with solving complex reasoning problems. These problems test various cognitive abilities including logical deduction, causal reasoning, mathematical thinking, and spatial understanding.
+
+    Instructions:
+    1. Read the entire problem carefully, including all given conditions and rules
+    2. Pay attention to the specific type of reasoning required (logical, temporal, spatial, etc.)
+    3. Consider all relationships and constraints mentioned in the problem
+    4. Apply structured thinking to reach a valid conclusion
+    5. Choose the answer that logically follows from the given information
+    6. Respond with ONLY the letter (A, B, C, etc.) or "True"/"False" or "Yes"/"No" - no explanations or additional text
 """
 )

pyproject.toml

@@ -67,3 +67,7 @@ build-backend = "poetry.core.masonry.api"
 markers = [
     "integration: marks tests as integration tests (deselect with '-m \"not integration\"')"
 ]
+
+[tool.bandit]
+exclude_dirs = ["tests"]
+skips = ["B101"]

tests/test_bbh_parser.py

@@ -0,0 +1,160 @@
+import pytest
+
+from llmdataparser.bbh_parser import BBHDatasetParser, BBHParseEntry
+
+
+@pytest.fixture
+def bbh_parser():
+    """Create a BBH parser instance for testing."""
+    return BBHDatasetParser()
+
+
+@pytest.fixture
+def loaded_bbh_parser(bbh_parser):
+    """Create and load a BBH parser instance for testing."""
+    bbh_parser.load(task_name="reasoning_about_colored_objects", split="test")
+    return bbh_parser
+
+
+@pytest.fixture
+def sample_row():
+    """Create a sample BBH data row for testing."""
+    return {
+        "input": "What color is the sky on a clear day?\nA) Blue\nB) Green\nC) Red\nD) Yellow",
+        "target": "(A)",
+    }
+
+
+def test_bbh_parse_entry_creation_valid():
+    """Test valid creation of BBHParseEntry."""
+    entry = BBHParseEntry.create(
+        prompt="Test prompt",
+        answer="A",
+        raw_question="Test question",
+        raw_answer="(A)",
+        task_name="reasoning_about_colored_objects",
+    )
+    assert isinstance(entry, BBHParseEntry)
+    assert entry.prompt == "Test prompt"
+    assert entry.answer == "A"
+    assert entry.raw_question == "Test question"
+    assert entry.raw_answer == "(A)"
+    assert entry.task_name == "reasoning_about_colored_objects"
+
+
+def test_bbh_parser_initialization(bbh_parser):
+    """Test BBH parser initialization."""
+    assert bbh_parser._data_source == "lukaemon/bbh"
+    assert bbh_parser._default_task == "reasoning_about_colored_objects"
+    assert "boolean_expressions" in bbh_parser._task_names
+    assert "word_sorting" in bbh_parser._task_names
+    assert (
+        bbh_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/lukaemon/bbh"
+    )
+
+
+def test_load_dataset(loaded_bbh_parser):
+    """Test loading the dataset."""
+    assert loaded_bbh_parser.raw_data is not None
+    assert loaded_bbh_parser.split_names == ["test"]
+    assert loaded_bbh_parser._current_task == "reasoning_about_colored_objects"
+
+
+@pytest.mark.integration
+def test_full_parse_workflow(loaded_bbh_parser):
+    """Test the complete workflow of loading and parsing data."""
+    # Parse the test split
+    loaded_bbh_parser.parse(split_names="test", force=True)
+    parsed_data = loaded_bbh_parser.get_parsed_data
+
+    # Basic checks
+    assert len(parsed_data) > 0
+
+    # Check first entry structure
+    first_entry = parsed_data[0]
+    assert isinstance(first_entry, BBHParseEntry)
+    assert first_entry.task_name == "reasoning_about_colored_objects"
+    assert first_entry.answer.strip("()").isalpha()  # Should be a single letter
+    assert first_entry.prompt.startswith(loaded_bbh_parser._system_prompt)
+
+
+def test_process_entry(bbh_parser, sample_row):
+    """Test processing of a single BBH entry."""
+    entry = bbh_parser.process_entry(
+        sample_row, task_name="reasoning_about_colored_objects"
+    )
+
+    assert isinstance(entry, BBHParseEntry)
+    assert entry.answer == "A"  # Stripped from "(A)"
+    assert "What color is the sky" in entry.raw_question
+    assert entry.raw_answer == "(A)"
+    assert bbh_parser._system_prompt in entry.prompt
+    assert entry.task_name == "reasoning_about_colored_objects"
+
+
+@pytest.mark.parametrize("split_name", ["invalid_split", "wrong_split"])
+def test_parse_with_invalid_split(bbh_parser, split_name):
+    """Test parsing with invalid split names."""
+    bbh_parser.raw_data = {"train": [], "test": []}  # Mock data
+
+    with pytest.raises(
+        ValueError, match=f"Split '{split_name}' not found in the dataset"
+    ):
+        bbh_parser.parse(split_name)
+
+
+def test_parse_without_loaded_data(bbh_parser):
+    """Test parsing without loading data first."""
+    with pytest.raises(
+        ValueError, match="No data loaded. Please load the dataset first"
+    ):
+        bbh_parser.parse()
+
+
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        {"input": "Test question", "target": "(A)"},
+        {"input": "Test question", "target": "(B)"},
+        {"input": "Test question", "target": "(C)"},
+    ],
+)
+def test_answer_stripping(bbh_parser, test_case):
+    """Test stripping of parentheses from answers."""
+    entry = bbh_parser.process_entry(
+        test_case, task_name="reasoning_about_colored_objects"
+    )
+    assert entry.answer == test_case["target"].strip("()")
+    assert entry.raw_answer == test_case["target"]
+
+
+def test_parser_properties(bbh_parser):
+    """Test parser property getters."""
+    assert len(bbh_parser.task_names) > 0
+    assert bbh_parser.total_tasks == len(bbh_parser._task_names)
+    assert all(isinstance(task, str) for task in bbh_parser.task_names)
+
+
+def test_parser_string_representation(loaded_bbh_parser):
+    """Test string representation of parser."""
+    repr_str = str(loaded_bbh_parser)
+    assert "BBHDatasetParser" in repr_str
+    assert "lukaemon/bbh" in repr_str
+    assert "reasoning_about_colored_objects" in repr_str
+    assert "loaded" in repr_str
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize(
+    "task_name", ["boolean_expressions", "causal_judgement", "date_understanding"]
+)
+def test_different_tasks_parsing(bbh_parser, task_name):
+    """Test parsing different tasks of the dataset."""
+    bbh_parser.load(task_name=task_name, split="test")
+    bbh_parser.parse(split_names="test", force=True)
+    parsed_data = bbh_parser.get_parsed_data
+
+    assert len(parsed_data) > 0
+    assert all(entry.task_name == task_name for entry in parsed_data)
+    assert all(isinstance(entry.answer, str) for entry in parsed_data)

tests/test_mmlu_parser.py

@@ -0,0 +1,220 @@
+import pytest
+
+from llmdataparser.mmlu_parser import (
+    BaseMMLUDatasetParser,
+    MMLUParseEntry,
+    MMLUProDatasetParser,
+    MMLUProParseEntry,
+    MMLUReduxDatasetParser,
+    TMMLUPlusDatasetParser,
+)
+
+
+@pytest.fixture
+def base_parser():
+    """Create a base MMLU parser instance."""
+    return BaseMMLUDatasetParser()
+
+
+@pytest.fixture
+def redux_parser():
+    """Create a MMLU Redux parser instance."""
+    return MMLUReduxDatasetParser()
+
+
+@pytest.fixture
+def tmmlu_parser():
+    """Create a TMMLU+ parser instance."""
+    return TMMLUPlusDatasetParser()
+
+
+@pytest.fixture
+def mmlu_pro_parser():
+    """Create a MMLU Pro parser instance."""
+    return MMLUProDatasetParser()
+
+
+@pytest.fixture
+def sample_mmlu_entries():
+    """Create sample MMLU dataset entries for testing."""
+    return [
+        {
+            "question": "What is the capital of France?",
+            "choices": ["London", "Paris", "Berlin", "Madrid"],
+            "answer": 1,  # Paris
+            "subject": "geography",
+        },
+        {
+            "question": "Which of these is a primary color?",
+            "choices": ["Green", "Purple", "Blue", "Orange"],
+            "answer": 2,  # Blue
+            "subject": "art",
+        },
+    ]
+
+
+@pytest.fixture
+def sample_mmlu_pro_entries():
+    """Create sample MMLU Pro dataset entries for testing."""
+    return [
+        {
+            "question": "What is the time complexity of quicksort?",
+            "options": ["O(n)", "O(n log n)", "O(n²)", "O(2ⁿ)", "O(n!)", "O(1)"],
+            "answer": "The average time complexity of quicksort is O(n log n)",
+            "answer_index": 1,
+            "category": "computer_science",
+        }
+    ]
+
+
+def test_mmlu_parse_entry_creation_valid():
+    """Test valid creation of MMLUParseEntry."""
+    entry = MMLUParseEntry.create(
+        prompt="Test prompt",
+        answer="A",
+        raw_question="Test question",
+        raw_choices=["choice1", "choice2", "choice3", "choice4"],
+        raw_answer="0",
+        task_name="test_task",
+    )
+    assert isinstance(entry, MMLUParseEntry)
+    assert entry.prompt == "Test prompt"
+    assert entry.answer == "A"
+    assert entry.raw_choices == ["choice1", "choice2", "choice3", "choice4"]
+    assert entry.task_name == "test_task"
+
+
+@pytest.mark.parametrize("invalid_answer", ["E", "F", "1", "", None])
+def test_mmlu_parse_entry_creation_invalid(invalid_answer):
+    """Test invalid answer handling in MMLUParseEntry creation."""
+    with pytest.raises(
+        ValueError, match="Invalid answer_letter.*must be one of A, B, C, D"
+    ):
+        MMLUParseEntry.create(
+            prompt="Test prompt",
+            answer=invalid_answer,
+            raw_question="Test question",
+            raw_choices=["choice1", "choice2", "choice3", "choice4"],
+            raw_answer="4",
+            task_name="test_task",
+        )
+
+
+def test_process_entry_base(base_parser, sample_mmlu_entries):
+    """Test processing entries in base MMLU parser."""
+    entry = base_parser.process_entry(sample_mmlu_entries[0], task_name="geography")
+
+    assert isinstance(entry, MMLUParseEntry)
+    assert entry.answer == "B"  # Index 1 maps to B
+    assert "A. London" in entry.prompt
+    assert "B. Paris" in entry.prompt
+    assert "C. Berlin" in entry.prompt
+    assert "D. Madrid" in entry.prompt
+    assert entry.raw_question == "What is the capital of France?"
+    assert entry.raw_choices == ["London", "Paris", "Berlin", "Madrid"]
+    assert entry.raw_answer == 1
+    assert entry.task_name == "geography"
+
+
+def test_mmlu_pro_parse_entry_creation_valid():
+    """Test valid creation of MMLUProParseEntry."""
+    entry = MMLUProParseEntry.create(
+        prompt="Test prompt",
+        answer="E",  # MMLU Pro supports up to J
+        raw_question="Test question",
+        raw_choices=["choice1", "choice2", "choice3", "choice4", "choice5"],
+        raw_answer="4",
+        task_name="test_task",
+    )
+    assert isinstance(entry, MMLUProParseEntry)
+    assert entry.answer == "E"
+    assert len(entry.raw_choices) == 5
+
+
+def test_process_entry_mmlu_pro(mmlu_pro_parser, sample_mmlu_pro_entries):
+    """Test processing entries in MMLU Pro parser."""
+    entry = mmlu_pro_parser.process_entry(
+        sample_mmlu_pro_entries[0], task_name="computer_science"
+    )
+
+    assert isinstance(entry, MMLUProParseEntry)
+    assert entry.answer == "B"  # Index 1 maps to B
+    assert "O(n log n)" in entry.prompt
+    assert entry.task_name == "computer_science"
+    assert len(entry.raw_choices) == 6
+
+
+def test_tmmlu_process_entry(tmmlu_parser):
+    """Test processing entries in TMMLU+ parser."""
+    test_row = {
+        "question": "什麼是台灣最高的山峰？",
+        "A": "玉山",
+        "B": "阿里山",
+        "C": "合歡山",
+        "D": "雪山",
+        "answer": "A",
+        "subject": "geography_of_taiwan",
+    }
+
+    entry = tmmlu_parser.process_entry(test_row, task_name="geography_of_taiwan")
+    assert isinstance(entry, MMLUParseEntry)
+    assert entry.answer == "A"
+    assert entry.raw_choices == ["玉山", "阿里山", "合歡山", "雪山"]
+    assert entry.task_name == "geography_of_taiwan"
+
+
+@pytest.mark.parametrize(
+    "parser_fixture,expected_tasks,expected_source",
+    [
+        ("base_parser", 57, "cais/mmlu"),
+        ("redux_parser", 30, "edinburgh-dawg/mmlu-redux"),
+        ("tmmlu_parser", 66, "ikala/tmmluplus"),
+        ("mmlu_pro_parser", 14, "TIGER-Lab/MMLU-Pro"),
+    ],
+)
+def test_parser_initialization(
+    request, parser_fixture, expected_tasks, expected_source
+):
+    """Test initialization of different MMLU parser variants."""
+    parser = request.getfixturevalue(parser_fixture)
+    assert len(parser.task_names) == expected_tasks
+    assert parser._data_source == expected_source
+    assert (
+        parser.get_huggingface_link
+        == f"https://huggingface.co/datasets/{expected_source}"
+    )
+
+
+@pytest.mark.integration
+def test_load_dataset(base_parser):
+    """Test loading the MMLU dataset."""
+    base_parser.load(task_name="anatomy", split="test")
+    assert base_parser.raw_data is not None
+    assert base_parser.split_names == ["test"]
+    assert base_parser._current_task == "anatomy"
+
+
+def test_parser_string_representation(base_parser):
+    """Test string representation of MMLU parser."""
+    repr_str = str(base_parser)
+    assert "MMLUDatasetParser" in repr_str
+    assert "cais/mmlu" in repr_str
+    assert "not loaded" in repr_str
+
+
+@pytest.mark.integration
+def test_different_splits_parsing(base_parser):
+    """Test parsing different splits of the dataset."""
+    # Load and parse test split
+    base_parser.load(task_name="anatomy", split="test")
+    base_parser.parse(split_names="test", force=True)
+    test_count = len(base_parser.get_parsed_data)
+
+    # Load and parse validation split
+    base_parser.load(task_name="anatomy", split="validation")
+    base_parser.parse(split_names="validation", force=True)
+    val_count = len(base_parser.get_parsed_data)
+
+    assert test_count > 0
+    assert val_count > 0
+    assert test_count != val_count