Update handler.py

handler.py

@@ -25,7 +25,8 @@ class EndpointHandler:
                 config,
                 device_map="auto",
                 torch_dtype=torch.float16,
-                trust_remote_code=True
+                trust_remote_code=True,
+                load_in_8bit=True # Load model in the lowest 4-bit precision quantization
             )
             
             max_memory = get_balanced_memory(
@@ -52,7 +53,8 @@ class EndpointHandler:
 
         with torch.autocast('cuda'):
             # preprocess
-            inputs = self.tokenizer(inputs, return_tensors="pt").to(self.device)
+            # inputs = self.tokenizer(inputs, return_tensors="pt").to(self.device)
+            inputs = self.tokenizer(inputs, return_tensors="pt")
     
             # pass inputs with all kwargs in data
             if parameters is not None: