Updated container + instance type

main_backend_lighteval.py

@@ -1,10 +1,6 @@
 import logging
 import pprint
 
-from huggingface_hub import snapshot_download
-
-logging.getLogger("openai").setLevel(logging.WARNING)
-
 import lighteval.models.endpoint_model
 class GoodInferenceEndpointModel(lighteval.models.endpoint_model.InferenceEndpointModel):
     def add_special_tokens(self):
@@ -17,15 +13,21 @@ def patched_init(self, *args, **kwargs):
     self.name = self.name.replace('.', '-')
 lighteval.models.endpoint_model.InferenceEndpointModelConfig.__init__ = patched_init
 
-# import huggingface_hub
-# orig_create_endpoint = huggingface_hub.create_inference_endpoint
-# def new_create_endpoint(*args, **kwargs):
-#     if 'custom_image' in kwargs and kwargs['custom_image']['url'] == "ghcr.io/huggingface/text-generation-inference:1.1.0":
-#         kwargs['custom_image']['url'] = "ghcr.io/huggingface/text-generation-inference:1.4.1"
-#     return orig_create_endpoint(*args, **kwargs)
-# huggingface_hub.create_inference_endpoint = new_create_endpoint
+import huggingface_hub
+orig_create_endpoint = huggingface_hub.create_inference_endpoint
+def new_create_endpoint(*args, **kwargs):
+    print('$$$$$$$$$$$$$$$$$ here 1')
+    if 'custom_image' in kwargs and kwargs['custom_image']['url'] == "ghcr.io/huggingface/text-generation-inference:1.1.0":
+        print('$$$$$$$$$$$$$ here 2')
+        kwargs['custom_image']['url'] = "registry.internal.huggingface.tech/api-inference/community/text-generation-inference:gemma-ie"
+    return orig_create_endpoint(*args, **kwargs)
+huggingface_hub.create_inference_endpoint = new_create_endpoint
 
 
+from huggingface_hub import snapshot_download
+
+logging.getLogger("openai").setLevel(logging.WARNING)
+
 from src.backend.run_eval_suite_lighteval import run_evaluation
 from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
 from src.backend.sort_queue import sort_models_by_priority
@@ -84,7 +86,8 @@ def run_auto_eval():
     # This needs to be done
     #instance_size, instance_type = get_instance_for_model(eval_request)
     # For GPU
-    instance_size, instance_type = "small", "g4dn.xlarge" 
+    # instance_size, instance_type = "small", "g4dn.xlarge" 
+    instance_size, instance_type = "medium", "g5.2xlarge" 
     # For CPU
     # instance_size, instance_type = "medium", "c6i"