add data

README.md

@@ -1,12 +1,13 @@
 ---
-title: Open Medical Llm Leaderboard
-emoji: 📊
+title: Test Leaderboard
+emoji: 🐢
 colorFrom: pink
-colorTo: green
+colorTo: red
 sdk: gradio
-sdk_version: 4.19.1
+sdk_version: 4.15.0
 app_file: app.py
 pinned: false
+license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -39,7 +39,7 @@ from src.display.utils import Tasks
 
 from huggingface_hub import snapshot_download
 
-## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## -------##
+## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- ## ------- 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
@@ -100,6 +100,8 @@ def filter_queries(query: str, filtered_df: pd.DataFrame):
 
 def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool) -> pd.DataFrame:
 
+
+    print("aa this is an example", df)
     print(f"filter_models()'s df: {df}\n")
     # Show all models
     if show_deleted:
@@ -108,7 +110,10 @@ def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precisio
         filtered_df = df[df[AutoEvalColumn.still_on_hub.name] is True]
 
     type_emoji = [t[0] for t in type_query]
+    print("aa this is an example", df, AutoEvalColumn.model_type_symbol.name, "thhhthht")
+    print("type", type_emoji)
     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+    print("bb", filtered_df)
     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
 
     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
@@ -353,7 +358,7 @@ scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=6 * 60 * 60)
 
 scheduler.start()
-# demo.queue(default_concurrency_limit=40).launch()
+demo.queue().launch()
 
 # demo.launch(show_api=False, enable_queue=False)
-demo.launch() # TypeError: Blocks.launch() got an unexpected keyword argument 'enable_queue'
+# demo.launch(enable_queue=False).queue() # TypeError: Blocks.launch() got an unexpected keyword argument 'enable_queue'

app_empty.py

@@ -4,5 +4,4 @@ def greet(name):
     return "Hello " + name + "!!"
 
 # iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-# iface.launch()
-# autocomplete
+# iface.launch()

requirements.txt

@@ -22,6 +22,7 @@ accelerate
 sentencepiece
 langdetect
 sacrebleu
+cchardet
 rouge_score
 bert-score
 evaluate

src/backend/envs.py

@@ -27,7 +27,7 @@ class Tasks(Enum):
     task5        = Task("college_medicine (mmlu)", "MMLU College Medicine", 0) 
     task6        = Task("medical_genetics (mmlu)", "MMLU Medical Genetics", 0) 
     task7        = Task("professional_medicine (mmlu)", "MMLU Professional Medicine", 0) 
-    task8       = Task("pubmedqa", "PubMedQA", 0) 
+    task8       = Task("pubmedqa", "PubMedQA", 0)
     
 
 

src/backend/run_eval_suite.py

@@ -33,7 +33,7 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
     # indexes all tasks from the `lm_eval/tasks` subdirectory.
     # Alternatively, you can set `TaskManager(include_path="path/to/my/custom/task/configs")`
     # to include a set of tasks in a separate directory.
-    task_manager = TaskManager(include_path="src/backend/open_medical_llm_leaderboard_tasks")
+    task_manager = TaskManager(include_path="src/backend/probing_tasks")
 
     if "gpt" in eval_request.model:
         model = "openai-chat-completions"

src/display/utils.py

@@ -1,13 +1,11 @@
-from dataclasses import dataclass, field, make_dataclass
+from dataclasses import dataclass, make_dataclass
 from enum import Enum
 
 import pandas as pd
 
 
 def fields(raw_class):
-    return [
-        v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
-    ]
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
 @dataclass
@@ -28,13 +26,10 @@ class Tasks(Enum):
     mmlu_mg = Task("medical_genetics (mmlu)", "acc", "MMLU Medical Genetics")
     mmlu_pm = Task("professional_medicine (mmlu)", "acc", "MMLU Professional Medicine")
     pubmedqa = Task("pubmedqa", "acc", "PubMedQA")
-
-
+    
 # These classes are for user facing column names,
 # to avoid having to change them all around the code
 # when a modif is needed
-
-
 @dataclass
 class ColumnContent:
     name: str
@@ -45,103 +40,29 @@ class ColumnContent:
     dummy: bool = False
     is_task: bool = False
 
-
-# Define a function to generate ColumnContent instances
-def column_content_factory(
-    name: str,
-    type: str,
-    displayed_by_default: bool,
-    hidden: bool = False,
-    never_hidden: bool = False,
-    dummy: bool = False,
-    is_task: bool = False,
-):
-    return lambda: ColumnContent(
-        name=name,
-        type=type,
-        displayed_by_default=displayed_by_default,
-        hidden=hidden,
-        never_hidden=never_hidden,
-        dummy=dummy,
-        is_task=is_task,
-    )
-
-
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(
-    [
-        "model_type_symbol",
-        ColumnContent,
-        ColumnContent("T", "str", True, never_hidden=True),
-    ]
-)
-auto_eval_column_dict.append(
-    [
-        "model",
-        ColumnContent,
-        ColumnContent("Model", "markdown", True, never_hidden=True),
-    ]
-)
-# Scores
-auto_eval_column_dict.append(
-    ["average", ColumnContent, ColumnContent("Avg", "number", True)]
-)
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg", "number", True)])
 for task in Tasks:
-    auto_eval_column_dict.append(
-        [
-            task.name,
-            ColumnContent,
-            ColumnContent(task.value.col_name, "number", True, is_task=True),
-        ]
-    )  # hidden was true by default
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, is_task=True)]) # hidden was true by default
 # Model information
-auto_eval_column_dict.append(
-    ["model_type", ColumnContent, ColumnContent("Type", "str", False)]
-)
-auto_eval_column_dict.append(
-    ["architecture", ColumnContent, ColumnContent("Architecture", "str", False)]
-)
-auto_eval_column_dict.append(
-    ["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)]
-)
-auto_eval_column_dict.append(
-    ["precision", ColumnContent, ColumnContent("Precision", "str", False)]
-)
-auto_eval_column_dict.append(
-    ["license", ColumnContent, ColumnContent("Hub License", "str", False)]
-)
-auto_eval_column_dict.append(
-    ["params", ColumnContent, ColumnContent("#Params (B)", "number", False)]
-)
-auto_eval_column_dict.append(
-    ["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)]
-)
-auto_eval_column_dict.append(
-    [
-        "still_on_hub",
-        ColumnContent,
-        ColumnContent("Available on the hub", "bool", False),
-    ]
-)
-auto_eval_column_dict.append(
-    ["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]
-)
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 # Dummy column for the search bar (hidden by the custom CSS)
-# Define the structure of your dataclass fields with default_factory for mutable defaults
-auto_eval_column_fields = [
-    (
-        "model_type_symbol",
-        ColumnContent,
-        field(
-            default_factory=column_content_factory("T", "str", True, never_hidden=True)
-        ),
-    ),
-    # Add other fields similarly...
-]
+auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_fields, frozen=True)
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
 
 @dataclass(frozen=True)
@@ -189,6 +110,9 @@ class WeightType(Enum):
     Delta = ModelDetails("Delta")
 
 
+
+
+
 class Precision(Enum):
     float32 = ModelDetails("float32")
     float16 = ModelDetails("float16")
@@ -213,17 +137,13 @@ class Precision(Enum):
         if precision in ["GPTQ", "None"]:
             return Precision.qt_GPTQ
         return Precision.Unknown
-
+        
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [
-    c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
-]
-TYPES_LITE = [
-    c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
-]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]

src/envs.py

@@ -5,6 +5,7 @@ from huggingface_hub import HfApi
 
 H4_TOKEN = os.environ.get("HF_SECRET", None) 
 
+# REPO_ID = "pminervini/hallucinations-leaderboard"
 REPO_ID = "openlifescienceai/open_medical_llm_leaderboard" 
 
 QUEUE_REPO = "openlifescienceai/test_requests"
@@ -16,7 +17,7 @@ PRIVATE_RESULTS_REPO = "openlifescienceai/test_private-results"
 
 IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
 
-
+# CACHE_PATH = "/Users/chaeeunlee/Documents/VSC_workspaces/test_leaderboard" # 
 CACHE_PATH = os.getenv("HF_HOME", ".") 
 
 print(f"CACHE_PATH = {CACHE_PATH}")
@@ -27,6 +28,7 @@ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
 EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
 EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
 
+# PATH_TO_COLLECTION = "hallucinations-leaderboard/llm-leaderboard-best-models-652d6c7965a4619fb5c27a03" # ??
 
 # Rate limit variables
 RATE_LIMIT_PERIOD = 7
@@ -34,4 +36,4 @@ RATE_LIMIT_QUOTA = 5
 HAS_HIGHER_RATE_LIMIT = ["TheBloke"]
 
 API = HfApi(token=H4_TOKEN)
-# API = HfApi()
+# API = HfApi()