Update README.md

README.md

@@ -45,10 +45,15 @@ quantization_config = BitsAndBytesConfig(
     bnb_4bit_quant_type="nf4"
     )
 
+
+
 model_id = "Ertugrul/Pixtral-12B-Captioner-Relaxed"
 model = LlavaForConditionalGeneration.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16)
 processor = AutoProcessor.from_pretrained(model_id)
 
+# for quantization just use this instead of previous load
+# model = LlavaForConditionalGeneration.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
+
 conversation = [
     {
         "role": "user",
@@ -81,6 +86,10 @@ def resize_image(image, target_size=768):
 # you can try different resolutions or disable it completely
 image = resize_image(image, 768)
 
+
+inputs = processor(text=PROMPT, images=image, return_tensors="pt").to("cuda")
+
+
 with torch.no_grad():
     with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
         generate_ids = model.generate(**inputs, max_new_tokens=384, do_sample=True, temperature=0.3, use_cache=True, top_k=20)