added example for parameters

main_backend_lighteval.py

@@ -5,7 +5,7 @@ from huggingface_hub import snapshot_download
 
 logging.getLogger("openai").setLevel(logging.WARNING)
 
-from backend.run_eval_suite_lighteval import run_evaluation
+from src.backend.run_eval_suite_lighteval import run_evaluation
 from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
 from src.backend.sort_queue import sort_models_by_priority
 
@@ -61,7 +61,8 @@ def run_auto_eval():
     )
 
     # This needs to be done
-    instance_size, instance_type = get_instance_for_model(eval_request)
+    #instance_size, instance_type = get_instance_for_model(eval_request)
+    instance_size, instance_type = "small", "g4dn.xlarge" 
 
     run_evaluation(
         eval_request=eval_request, 

src/envs.py

@@ -13,9 +13,9 @@ DEVICE = "cpu" # "cuda:0" if you add compute, for harness evaluations
 LIMIT = 20 # !!!! Should be None for actual evaluations!!!
 
 # For lighteval evaluations
-ACCELERATOR = ""
-REGION = ""
-VENDOR = ""
+ACCELERATOR = "gpu"
+REGION = "us-east-1"
+VENDOR = "aws"
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard-backend"