shalomma
/

llama-7b-custom

Inference Endpoints

Model card Files Files and versions Community

shalomma commited on Mar 6, 2023

Commit

801051f

·

1 Parent(s): 77b6161

drop 8 bit arg

Files changed (1) hide show

handler.py +1 -1

handler.py CHANGED Viewed

@@ -11,7 +11,7 @@ MODEL = 'decapoda-research/llama-7b-hf'
 class EndpointHandler():
     def __init__(self, path=""):
         self.tokenizer = llama.LLaMATokenizer.from_pretrained(MODEL)
-        self.model = llama.LLaMAForCausalLM.from_pretrained(MODEL, low_cpu_mem_usage=True, load_in_8bit=True)
         self.model.to('cuda')
     def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:

 class EndpointHandler():
     def __init__(self, path=""):
         self.tokenizer = llama.LLaMATokenizer.from_pretrained(MODEL)
+        self.model = llama.LLaMAForCausalLM.from_pretrained(MODEL, low_cpu_mem_usage=True)
         self.model.to('cuda')
     def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]: