Adding torch_dtype = float16
Browse files- handler.py +1 -1
handler.py
CHANGED
@@ -22,7 +22,7 @@ class EndpointHandler():
|
|
22 |
|
23 |
self.model = Blip2ForConditionalGeneration.from_pretrained(
|
24 |
"Salesforce/blip2-flan-t5-xxl", device_map=device_map,
|
25 |
-
|
26 |
load_in_8bit=True, load_in_8bit_fp32_cpu_offload=True
|
27 |
)
|
28 |
|
|
|
22 |
|
23 |
self.model = Blip2ForConditionalGeneration.from_pretrained(
|
24 |
"Salesforce/blip2-flan-t5-xxl", device_map=device_map,
|
25 |
+
torch_dtype=torch.float16,
|
26 |
load_in_8bit=True, load_in_8bit_fp32_cpu_offload=True
|
27 |
)
|
28 |
|