Changed transformer version to fix issues.

requirements.txt

@@ -1,14 +1,15 @@
 -e git+https://github.com/facebookresearch/audiocraft.git@f83babff6b5e97f75562127c4cc8122229c8f099#egg=audiocraft
 phonemizer
+transformers==4.43.1
+torch==2.1.1
+numpy==2.0.1
+torchaudio
+torchvision
 spaces
 huggingface-hub
 num2words
-transformers
 numpy
 pillow
 safetensors
 tokenizers
-torch==2.1.0
-torchaudio
-torchvision
 aeneas

src/model/modules/imagecraft.py

@@ -405,15 +405,10 @@ class ImageCraft(nn.Module):
         max_tokens=30,
         do_sample=False,
         output_type="file",
-        return_output="speech",
     ):
-        if return_output == "speech" or return_output is None:
-            transcript = self._generate_caption(image, max_tokens, do_sample)
-            speech = self._generate_speech(transcript, output_type)
-            return transcript, speech
-        else:
-            transcript = self._generate_caption(image, max_tokens, do_sample)
-            return transcript
+        transcript = self._generate_caption(image, max_tokens, do_sample)
+        speech = self._generate_speech(transcript, output_type)
+        return transcript, speech
 
     @classmethod
     def from_pretrained(

src/model/modules/imagecraftprocessor.py

@@ -40,9 +40,6 @@ class ImageCraftProcessor:
         tokenizer.add_eos_token = False
 
         self.tokenizer = tokenizer
-        # self.image_processor = SiglipImageProcessor.from_pretrained(
-        #     "google/siglip-so400m-patch14-384"
-        # )
 
     def __call__(
         self,
@@ -55,9 +52,6 @@ class ImageCraftProcessor:
             len(images) == 1 and len(text) == 1
         ), f"Received {len(images)} images for {len(text)} prompts."
 
-        # pixel_values = self.image_processor(images=images, return_tensors="pt")[
-        #     "pixel_values"
-        # ]
         pixel_values = process_images(
             images,
             size=(self.image_size, self.image_size),

src/utils/model_utils.py

@@ -19,13 +19,11 @@ def get_model_inputs(
     processor: ImageCraftProcessor,
     prompt: str,
     image: Image,
-    suffix: Optional[str] = None,
     device: str = "cuda",
 ):
     images = [image]
     prompts = [prompt]
-    if suffix is not None:
-        suffix = [suffix]
+
     model_inputs = processor(text=prompts, images=images)
     model_inputs = move_inputs_to_device(model_inputs, device)
     return model_inputs
@@ -38,36 +36,3 @@ def get_config(config_file="config.json"):
         config = ImageCraftConfig(**model_config_file)
 
     return config
-
-
-# def load_hf_model(model_path: str, device: str) -> Tuple[ImageCraft, AutoTokenizer]:
-
-#     # Load the tokenizer
-#     tokenizer = AutoTokenizer.from_pretrained(model_path, padding_side="right")
-#     assert tokenizer.padding_side == "right"
-
-#     # Find all the *.safetensors files
-#     safetensors_files = glob.glob(os.path.join(model_path, "*.safetensors"))
-
-#     # ... and load them one by one in the tensors dictionary
-#     tensors = {}
-#     for safetensors_file in safetensors_files:
-#         with safe_open(safetensors_file, framework="pt", device="cpu") as f:
-#             for key in f.keys():
-#                 tensors[key] = f.get_tensor(key)
-
-#     # Load the model's config
-#     with open(os.path.join(model_path, "config.json"), "r") as f:
-#         model_config_file = json.load(f)
-#         config = ImageCraftConfig(**model_config_file)
-
-#     # Create the model using the configuration
-#     model = ImageCraft(config).to(device)
-
-#     # Load the state dict of the model
-#     model.load_state_dict(tensors, strict=False)
-
-#     # Tie weights
-#     model.tie_weights()
-
-#     return (model, tokenizer)