Update handler.py

handler.py

@@ -1,31 +1,34 @@
 from typing import Dict
-from faster_whisper import WhisperModel
+from faster_whisper import WhisperModel, Streaming
 import io
+import re
 
 class EndpointHandler:
     def __init__(self, model_dir=None):
-        # Set model size, assuming installation has been done with appropriate model files and setup
+        # Use int8 on CPU to reduce memory usage and potentially increase speed.
+        compute_type = "int8" if model_dir == "cpu" else "float16"
+
+        # Initialize WhisperModel with given model_size and compute_type
         model_size = "medium" if model_dir is None else model_dir
-        # Change to 'cuda' to use the GPU, and set compute_type for faster computation
-        self.model = WhisperModel(model_size, device="cuda", compute_type="float16")
+        self.model = WhisperModel(model_size, device=model_dir, compute_type=compute_type)
 
     def __call__(self, data: Dict) -> Dict[str, str]:
-        # Process the input data expected to be in 'inputs' key containing audio file bytes
         audio_bytes = data["inputs"]
-        
-        # Convert bytes to a file-like object
         audio_file = io.BytesIO(audio_bytes)
-        
-        # Perform transcription using the model
-        segments, info = self.model.transcribe(audio_file)
-        
-        # Compile the results into a text string and extract language information
-        # Strip whitespace from each segment before joining them
+
+        # Use Streaming interface to leverage VAD and potential speed improvements.
+        # Small beam size to speed up transcription. Adjust based on performance/accuracy needs.
+        beam_size = 1
+        streaming = Streaming(device=model_dir, compute_type=compute_type, vad=True)
+        segments, info = streaming.transcribe(audio_file, beam_size=beam_size)
+
+        # Aggregate transcribed text and remove any extra spaces.
         text = " ".join(segment.text.strip() for segment in segments)
+        text = re.sub(' +', ' ', text)
+
         language_code = info.language
         language_prob = info.language_probability
 
-        # Compile the response dictionary
         result = {
             "text": text,
             "language": language_code,