update model init with float16

README.md

@@ -27,7 +27,7 @@ Resources and Technical Documentation:
  
 ## How to Get Started with the Model
 
-Use the code below to get started with the model.
+Use the code below to get started with the model.  All models are trained with float16. 
 
 ```python
 import requests
@@ -35,8 +35,10 @@ import requests
 from PIL import Image
 from transformers import AutoProcessor, AutoModelForCausalLM 
 
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
 
-model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-base-ft", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-base-ft", torch_dtype=torch_dtype, trust_remote_code=True).to(device)
 processor = AutoProcessor.from_pretrained("microsoft/Florence-2-base-ft", trust_remote_code=True)
 
 prompt = "<OD>"
@@ -44,7 +46,7 @@ prompt = "<OD>"
 url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg?download=true"
 image = Image.open(requests.get(url, stream=True).raw)
 
-inputs = processor(text=prompt, images=image, return_tensors="pt")
+inputs = processor(text=prompt, images=image, return_tensors="pt").to(device, torch_dtype)
 
 generated_ids = model.generate(
     input_ids=inputs["input_ids"],
@@ -77,8 +79,10 @@ import requests
 from PIL import Image
 from transformers import AutoProcessor, AutoModelForCausalLM 
 
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
 
-model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-base-ft", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-base-ft", torch_dtype=torch_dtype, trust_remote_code=True).to(device)
 processor = AutoProcessor.from_pretrained("microsoft/Florence-2-base-ft", trust_remote_code=True)
 
 url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg?download=true"
@@ -89,7 +93,7 @@ def run_example(task_prompt, text_input=None):
         prompt = task_prompt
     else:
         prompt = task_prompt + text_input
-    inputs = processor(text=prompt, images=image, return_tensors="pt")
+    inputs = processor(text=prompt, images=image, return_tensors="pt").to(device, torch_dtype)
     generated_ids = model.generate(
       input_ids=inputs["input_ids"],
       pixel_values=inputs["pixel_values"],

config.json

@@ -79,7 +79,7 @@
     "image_feature_source": ["spatial_avg_pool", "temporal_avg_pool"]
   },
   "vocab_size": 51289,
-  "torch_dtype": "float32",
+  "torch_dtype": "float16",
   "transformers_version": "4.41.0.dev0",
   "is_encoder_decoder": true
 }