fix

app.py

@@ -1,5 +1,7 @@
 import logging
 import os
+os.makedirs("tmp", exist_ok=True)
+os.environ['TMP_DIR'] = "tmp"
 import subprocess
 
 import gradio as gr
@@ -22,7 +24,7 @@ from src.display.utils import (
 from src.envs import API, H4_TOKEN, HF_HOME, REPO_ID, RESET_JUDGEMENT_ENV
 from src.leaderboard.build_leaderboard import build_leadearboard_df, download_openbench, download_dataset
 import huggingface_hub
-huggingface_hub.login(token=H4_TOKEN)
+# huggingface_hub.login(token=H4_TOKEN)
 
 os.environ["GRADIO_ANALYTICS_ENABLED"] = "false"
 
@@ -126,16 +128,19 @@ def update_board():
     os.environ[RESET_JUDGEMENT_ENV] = "0"
     import shutil
     shutil.rmtree("m_data")
+    shutil.rmtree("data")
     download_dataset("Vikhrmodels/s-openbench-eval", "m_data")
     import glob
-    data_list = []
-    for file in glob.glob("m_data/model_data/external/*.json"):
+    data_list = [{"musicmc": 0.3021276595744681, "lawmc": 0.2800829875518672, "model": "apsys/saiga_3_8b", "moviesmc": 0.3472222222222222, "booksmc": 0.2800829875518672, "model_dtype": "torch.float16", "ppl": 0}]
+    for file in glob.glob("./m_data/model_data/external/*.json"):
         with open(file) as f:
             try:
                 data = json.load(f)
                 data_list.append(data)
             except:
                 continue
+    if len(data_list) >=1:
+        data_list.pop(0)
     with open("genned.json", "w") as f:
         json.dump(data_list, f)
 

data/leaderboard.json

@@ -1 +1 @@
-[{"musicmc": 0.3021276595744681, "lawmc": 0.2800829875518672, "model": "apsys/saiga_3_8b", "moviesmc": 0.3472222222222222, "booksmc": 0.2800829875518672, "model_dtype": "torch.float16", "ppl": 0}]
+[]

m_data/model_data/external/saiga_3_8bapsys.json

@@ -1 +1 @@
-{"musicmc": 0.3021276595744681, "lawmc": 0.2800829875518672, "model": "apsys/saiga_3_8b", "moviesmc": 0.3472222222222222, "booksmc": 0.2800829875518672, "model_dtype": "torch.float16", "ppl": 0}
+{"musicmc": 0.2936170212765957, "lawmc": 0.48094747682801237, "model": "apsys/saiga_3_8b", "moviesmc": 0.3402777777777778, "booksmc": 0.3112033195020747, "model_dtype": "torch.float16", "ppl": 0}

src/envs.py

@@ -16,7 +16,7 @@ PRIVATE_RESULTS_REPO = "open-llm-leaderboard/private-results"
 IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
 
 HF_HOME = os.getenv("HF_HOME", ".")
-HF_TOKEN_PRIVATE = os.environ.get("HF_TOKEN_PRIVATE")
+HF_TOKEN_PRIVATE = os.environ.get("H4_TOKEN")
 
 # Check HF_HOME write access
 print(f"Initial HF_HOME set to: {HF_HOME}")

src/leaderboard/build_leaderboard.py

@@ -35,6 +35,7 @@ def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, ba
             snapshot_download(
                 repo_id=repo_id,
                 local_dir=local_dir,
+                cache_dir='./tmp',
                 repo_type=repo_type,
                 tqdm_class=None,
                 token=HF_TOKEN_PRIVATE,
@@ -63,9 +64,10 @@ def build_leadearboard_df():
     # Retrieve the leaderboard DataFrame
     with open(f"{os.path.abspath(DATA_PATH)}/leaderboard.json", "r", encoding="utf-8") as eval_file:
         f=json.load(eval_file)
+        print(f)
+
         leaderboard_df = pd.DataFrame.from_records(f)[['model','moviesmc','musicmc','lawmc','booksmc','model_dtype','ppl']]
         leaderboard_df['avg'] = leaderboard_df[['moviesmc','musicmc','lawmc','booksmc']].mean(axis=1)
         numeric_cols = leaderboard_df.select_dtypes(include=['number']).columns
         leaderboard_df[numeric_cols] = leaderboard_df[numeric_cols].round(3)
-        print(f)
     return leaderboard_df.copy()