Added timeout redundancy, fixed naming for underscores

src/backend/run_eval_suite_lighteval.py

@@ -9,6 +9,7 @@ from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN, OWNER
 from src.backend.manage_requests import EvalRequest
 from lighteval.logging.evaluation_tracker import EnhancedJSONEncoder
 from lighteval.models.model_loader import ModelInfo
+from huggingface_hub.errors import InferenceEndpointTimeoutError
 
 logging.getLogger("openai").setLevel(logging.WARNING)
 
@@ -24,7 +25,7 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
             "model_config": dict(model=dict(
                 type="endpoint",
                 base_params=dict(
-                    endpoint_name=f'{eval_request.model.split("/")[1].replace(".", "-").lower()}-lighteval'[-32:].strip('-'),
+                    endpoint_name=f'{eval_request.model.split("/")[1].replace(".", "-").replace('_', '-').lower()}-lighteval'[-32:].strip('-'),
                     model=eval_request.model,
                     revision=eval_request.revision,
                     dtype=eval_request.precision,
@@ -61,7 +62,16 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
     })
 
     try:
-        results = main(args)
+        # in case of timeout, try it again with reuse_existing
+        for i in range(3):
+            try:
+                results = main(args)
+                # if we are i>0, then raise an error so that we call clean up 
+                if i > 0: raise Exception() 
+            except InferenceEndpointTimeoutError:
+                if i < 3: 
+                    print('Timed out, trying again...')
+                    args.model_config['model']['base_params']['reuse_existing'] = True
 
         dumped = json.dumps(results, cls=EnhancedJSONEncoder, indent=2)
         print(dumped)