updated main.py to support the use of a quant molmo

README.md

@@ -31,18 +31,19 @@ A 12GB GPU should be fine.
 3. install the dependencies
    ``` pip3 install -r requirements.txt ```
 4. run the gradio version:
-   ``` python3 main.py ```
+   ``` python3 main.py ``` (use original molmo model at bf16)
+   or
+    ``` python3 main.py -q``` (use 4bit quant molmo model)
    1. create a zip file of images
    2. upload it
    3. process it
    4. click the button to download the caption zip file, the link is at the top of the page
 
    run the command-line version:
-   ``` python3 caption.py ``` (use molmo at bf16 for more accuracy; needs 24GB GPU)
-   ``` python3 caption.py -q ``` (use molmo at int4; should be fine with 12GB GPU)
+   ``` python3 caption.py ``` (use original molmo model at bf16)
+   ``` python3 caption.py -q ``` (use 4bit quant molmo model)
    1. make sure your images are in the "images" directory
    2. captions will be placed in the "images" directory
    
 Note:
-- main.py (gradio version does not yet support quant model)
 - If torch sees your first GPU supports flash attention and the others do not, it will assume all the cards do and it will throw an exception. A workaround is to use, for example, "CUDA_VISIBLE_DEVICES=0 python3 main.py (or caption.py)", to force torch to ignore the card supporting flash attention, so that it will use your other cards without it. Or, use it to exclude non-flash-attention-supporting GPUs.

main.py

@@ -24,6 +24,11 @@ def cleanup_temp_files():
             if os.path.isdir(dir_path):
                 shutil.rmtree(dir_path)
 
+# Parse command-line arguments
+parser = argparse.ArgumentParser(description="Load and use a quantized model")
+parser.add_argument("-q", "--use_quant", action="store_true", help="Use quantized model")
+args = parser.parse_args()
+
 if torch.cuda.is_available():
     device = torch.device("cuda")
     print("GPU is available. Using CUDA.")
@@ -31,9 +36,8 @@ else:
     device = torch.device("cpu")
     print("GPU is not available. Using CPU.")
 
-# load the processor from local path
+# Load the processor
 local_path = "./model/Molmo-7B-D-0924"
-#print("Loading processor from local path...")
 processor = AutoProcessor.from_pretrained(
     local_path,
     local_files_only=True,
@@ -41,23 +45,26 @@ processor = AutoProcessor.from_pretrained(
     torch_dtype='auto',
     device_map='auto'
 )
-#print("Processor loaded.")
-
-print("Loading model from local path...")
-model = AutoModelForCausalLM.from_pretrained(
-    local_path,
-    trust_remote_code=True,
-    torch_dtype='auto',
-    device_map='auto',
-)
-#print("Model loaded.")
-
-generation_config = GenerationConfig(max_new_tokens=300, stop_strings="<|endoftext|>")
-bits_and_bytes_config = BitsAndBytesConfig()
 
-# load the model in bf16 to reduce VRAM needed
-model.to(dtype=torch.bfloat16)
-#print("Model loaded in bf16")
+# Load the model
+if args.use_quant:
+    # Load the quantized model
+    quantized_local_path = "./model/molmo-7B-D-bnb-4bit"
+    model = AutoModelForCausalLM.from_pretrained(
+        quantized_local_path,
+        trust_remote_code=True,
+        torch_dtype='auto',
+        device_map='auto',
+    )
+else:
+    # Load the non-quantized model
+    model = AutoModelForCausalLM.from_pretrained(
+        local_path,
+        trust_remote_code=True,
+        torch_dtype='auto',
+        device_map='auto',
+    )
+    model.to(dtype=torch.bfloat16)
 
 def unzip_images(zip_file):
     # Create a unique directory for extracted images inside the "images" directory