support downloading directly into model_answers

app.py

@@ -75,7 +75,7 @@ def build_demo():
                         logging.info("New submition: file saved to %s", file_path)
                         API.upload_file(
                             path_or_fileobj=file.name,
-                            path_in_repo="./external/" + file_path,
+                            path_in_repo="model_answers/external/" + file_path,
                             repo_id="Vikhrmodels/openbench-eval",
                             repo_type="dataset",
                         )

data/leaderboard_logs/README.md

@@ -0,0 +1,3 @@
+# Логи генерации leaderboard
+Сюда из space отправляются после генерации
+Сохраняется только последний за день

src/envs.py

@@ -27,7 +27,6 @@ if not os.access(HF_HOME, os.W_OK):
 else:
     print("Write access confirmed for HF_HOME")
 
-EVAL_RESULTS_PATH = os.path.join(HF_HOME, "openbench")
 DATA_ARENA_PATH = os.path.join(HF_HOME, "data/arena-hard-v0.1")
 
 RESET_JUDGEMENT_ENV = "RESET_JUDGEMENT"

src/gen/gen_answer.py

@@ -4,30 +4,29 @@ Usage:
 python gen_api_answer --parallel 32
 """
 import argparse
+import concurrent.futures
 import json
 import os
 import time
-import concurrent.futures
 
-import tiktoken
 import shortuuid
+import tiktoken
 import tqdm
-
 from utils import (
-    load_questions,
-    load_model_answers,
-    make_config,
-    get_endpoint,
-    chat_completion_openai,
-    chat_completion_yandex,
-    chat_completion_gigachat,
+    OPENAI_MODEL_LIST,
     chat_completion_anthropic,
-    chat_completion_openai_azure,
-    chat_completion_mistral,
-    chat_completion_gemini,
     chat_completion_cohere,
+    chat_completion_gemini,
+    chat_completion_gigachat,
+    chat_completion_mistral,
+    chat_completion_openai,
+    chat_completion_openai_azure,
+    chat_completion_yandex,
+    get_endpoint,
+    load_model_answers,
+    load_questions,
+    make_config,
     reorg_answer_file,
-    OPENAI_MODEL_LIST,
     temperature_config,
 )
 
@@ -139,7 +138,7 @@ if __name__ == "__main__":
     settings = make_config(args.setting_file)
     endpoint_list = make_config(args.endpoint_file)
 
-    existing_answer = load_model_answers(os.path.join("data", settings["bench_name"], "model_answer"))
+    existing_answer = load_model_answers(os.path.join("data", settings["bench_name"], "model_answers", "internal"))
 
     print(settings)
 
@@ -150,7 +149,7 @@ if __name__ == "__main__":
         question_file = os.path.join("data", settings["bench_name"], "question.jsonl")
         questions = load_questions(question_file)
 
-        answer_file = os.path.join("data", settings["bench_name"], "model_answer", f"{model}.jsonl")
+        answer_file = os.path.join("data", settings["bench_name"], "model_answers", f"{model}.jsonl")
         print(f"Output to {answer_file}")
 
         if "parallel" in endpoint_info:

src/gen/gen_judgment.py

@@ -143,8 +143,8 @@ if __name__ == "__main__":
         pattern = re.compile(configs["regex_pattern"])
 
     question_file = os.path.join("./data", configs["bench_name"], "question.jsonl")
-    external_dir = os.path.join("./data", configs["bench_name"], "model_answer/external")
-    internal_dir = os.path.join("./data", configs["bench_name"], "model_answer/internal")
+    external_dir = os.path.join("./data", configs["bench_name"], "model_answers/external")
+    internal_dir = os.path.join("./data", configs["bench_name"], "model_answers/internal")
     ref_answer_dir = os.path.join("data", configs["bench_name"], "reference_answer")
 
     questions = load_questions(question_file)
@@ -157,7 +157,7 @@ if __name__ == "__main__":
     # if user choose a set of models, only judge those models
     models = [
         model.split("/")[-1].split(".")[0]
-        for model in glob.glob("./data/arena-hard-v0.1/model_answer/external/*.jsonl")
+        for model in glob.glob("./data/arena-hard-v0.1/model_answers/external/*.jsonl")
     ]
 
     ref_answers = None

src/gen/show_result.py

@@ -196,7 +196,7 @@ if __name__ == "__main__":
         args.load_battles and args.load_bootstrap
     ), "If loading prexisting bootstrapping data, you must also load preexisting battles."
 
-    answer_dir = os.path.join("data", args.bench_name, "model_answer/external")
+    answer_dir = os.path.join("data", args.bench_name, "model_answers/external")
     model_answers = load_model_answers(answer_dir)
 
     if args.load_battles:
@@ -269,7 +269,7 @@ if __name__ == "__main__":
 
         huggingface_hub.HfApi().upload_file(
             path_or_fileobj=json_file_name,
-            path_in_repo=f"leaderboard_logs/{json_file_name}",
-            repo_id="Vikhrmodels/openbench-eval",
+            path_in_repo=f"data/leaderboard_logs/{json_file_name}",
+            repo_id="Vikhrmodels/leaderboard",
             repo_type="dataset",
         )

src/leaderboard/build_leaderboard.py

@@ -53,43 +53,27 @@ def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, ba
 
 def download_openbench():
     """Downloads pre generated data"""
-    os.makedirs(f"{DATA_ARENA_PATH}/model_answer/internal/", exist_ok=True)
-    os.makedirs(f"{DATA_ARENA_PATH}/model_answer/external/", exist_ok=True)
-    os.makedirs(f"{DATA_ARENA_PATH}/model_answer/model_judgement/", exist_ok=True)
-    os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
+    os.makedirs(DATA_ARENA_PATH, exist_ok=True)
 
     # download answers of different models that we trust
-    download_dataset("Vikhrmodels/openbench-eval", EVAL_RESULTS_PATH)
-
-    subprocess.run(["pwd"], check=False)
-    logging.info(EVAL_RESULTS_PATH)
-    print("\nFiles openbench-eval:")
-    subprocess.run(["ls", EVAL_RESULTS_PATH], check=False)
+    download_dataset("Vikhrmodels/openbench-eval", DATA_ARENA_PATH)
 
     print("\nInternal models in openbench-eval:")
-    subprocess.run(["ls", f"{EVAL_RESULTS_PATH}/internal/"], check=False)
+    subprocess.run(["ls", f"{DATA_ARENA_PATH}/model_answers/internal/"], check=False)
+
+    print("\External models in openbench-eval:")
+    subprocess.run(["ls", f"{DATA_ARENA_PATH}/model_answers/external/"], check=False)
 
     print("\nJudgement in openbench-eval")
-    subprocess.run(["ls", f"{EVAL_RESULTS_PATH}/model_judgment/gpt-4-1106-preview"], check=False)
+    subprocess.run(["ls", f"{DATA_ARENA_PATH}/model_judgment/gpt-4-1106-preview"], check=False)
 
-    # copy the trusted model answers to data
-    subprocess.run(
-        [
-            "rsync",
-            "-azPvh",
-            f"{EVAL_RESULTS_PATH}/internal/*",
-            f"{DATA_ARENA_PATH}/model_answer/internal/",
-        ],
-        check=True,
-    )
-    # copy the judgement pre generated
-    # Will be rewritten after we switch to new gen for each submit
+    # copy the model answers / judgements to data folder
     subprocess.run(
         [
             "rsync",
             "-azPvh",
-            f"{EVAL_RESULTS_PATH}/model_judgment/*",
-            f"{DATA_ARENA_PATH}/model_judgement/",
+            f"{EVAL_RESULTS_PATH}/*",
+            f"{DATA_ARENA_PATH}/",
         ],
         check=True,
     )