Transcribe non-speech areas too in "silero vad"

The old version is now in "silero-vad-skip-gaps". This
may introduce more noise in the transcript, but some of it
will be correct as well.

app.py

@@ -72,10 +72,17 @@ class UI:
 
                 # The results
                 if (vad == 'silero-vad'):
-                    # Use Silero VAD
+                    # Use Silero VAD and include gaps
                     if (self.vad_model is None):
-                        self.vad_model = VadSileroTranscription()
+                        self.vad_model = VadSileroTranscription(transcribe_non_speech= True)
                     result = self.vad_model.transcribe(source, whisperCallable)
+                elif (vad == 'silero-vad-skip-gaps'):
+                    # Use Silero VAD 
+                    if (self.vad_model is None):
+                        self.vad_model = VadSileroTranscription(transcribe_non_speech= True)
+                        
+                    skip_gaps = VadSileroTranscription(transcribe_non_speech = False, copy=self.vad_model)
+                    result = skip_gaps.transcribe(source, whisperCallable)
                 elif (vad == 'periodic-vad'):
                     # Very simple VAD - mark every 5 minutes as speech. This makes it less likely that Whisper enters an infinite loop, but
                     # it may create a break in the middle of a sentence, causing some artifacts.
@@ -184,7 +191,7 @@ def createUi(inputAudioMaxDuration, share=False, server_name: str = None):
         gr.Audio(source="upload", type="filepath", label="Upload Audio"), 
         gr.Audio(source="microphone", type="filepath", label="Microphone Input"),
         gr.Dropdown(choices=["transcribe", "translate"], label="Task"),
-        gr.Dropdown(choices=["none", "silero-vad", "periodic-vad"], label="VAD"),
+        gr.Dropdown(choices=["none", "silero-vad", "silero-vad-skip-gaps", "periodic-vad"], label="VAD"),
     ], outputs=[
         gr.File(label="Download"),
         gr.Text(label="Transcription"), 

vad.py

@@ -9,19 +9,24 @@ import torch
 import ffmpeg
 import numpy as np
 
-# Defaults
+from utils import format_timestamp
+
+# Defaults for Silero
 SPEECH_TRESHOLD = 0.3
 MAX_SILENT_PERIOD = 10 # seconds
 SEGMENT_PADDING_LEFT = 1 # Start detected text segment early
-SEGMENT_PADDING_RIGHT = 4 # End detected segments late
+SEGMENT_PADDING_RIGHT = 3 # End detected segments late
 
+# Whether to attempt to transcribe non-speech
+TRANSCRIBE_NON_SPEECH = False
 
 class AbstractTranscription(ABC):
-    def __init__(self, segment_padding_left: int = None, segment_padding_right = None, max_silent_period: int = None):
+    def __init__(self, segment_padding_left: int = None, segment_padding_right = None, max_silent_period: int = None, transcribe_non_speech: bool = False):
         self.sampling_rate = 16000
         self.segment_padding_left = segment_padding_left
         self.segment_padding_right = segment_padding_right
         self.max_silent_period = max_silent_period
+        self.transcribe_non_speech = transcribe_non_speech
 
     def get_audio_segment(self, str, start_time: str = None, duration: str = None):
         return load_audio(str, self.sampling_rate, start_time, duration)
@@ -68,6 +73,13 @@ class AbstractTranscription(ABC):
         print("Timestamps:")
         pprint(merged)
 
+        if self.transcribe_non_speech:
+            max_audio_duration = float(ffmpeg.probe(audio)["format"]["duration"])
+            merged = self.include_gaps(merged, min_gap_length=5, total_duration=max_audio_duration)
+
+            print("Transcribing non-speech:")
+            pprint(merged)
+
         result = {
             'text': "",
             'segments': [],
@@ -78,12 +90,19 @@ class AbstractTranscription(ABC):
         # For each time segment, run whisper
         for segment in merged:
             segment_start = segment['start']
-            segment_duration = segment['end'] - segment_start
+            segment_end = segment['end']
+            segment_gap = segment.get('gap', False)
+
+            segment_duration = segment_end - segment_start
 
             segment_audio = self.get_audio_segment(audio, start_time = str(segment_start) + "s", duration = str(segment_duration) + "s")
 
-            print("Running whisper on " + str(segment_start) + ", duration: " + str(segment_duration))
-            segment_result = whisperCallable(segment_audio)
+            print("Running whisper from ", format_timestamp(segment_start), " to ", format_timestamp(segment_end), ", duration: ", segment_duration, "gap: ", segment_gap)
+            if segment_gap:
+                # TODO: Use different parameters for these segments, as they are less likely to contain speech
+                segment_result = whisperCallable(segment_audio)
+            else:
+                segment_result = whisperCallable(segment_audio)
             adjusted_segments = self.adjust_whisper_timestamp(segment_result["segments"], adjust_seconds=segment_start, max_source_time=segment_duration)
 
             # Append to output
@@ -98,6 +117,32 @@ class AbstractTranscription(ABC):
 
         return result
             
+    def include_gaps(self, segments: Iterator[dict], min_gap_length: float, total_duration: float):
+        result = []
+        last_end_time = 0
+
+        for segment in segments:
+            segment_start = float(segment['start'])
+            segment_end = float(segment['end'])
+
+            if (last_end_time != segment_start):
+                delta = segment_start - last_end_time
+
+                if (min_gap_length is None or delta >= min_gap_length):
+                    result.append( { 'start': last_end_time, 'end': segment_start, 'gap': True } )
+            
+            last_end_time = segment_end
+            result.append(segment)
+
+        # Also include total duration if specified
+        if (total_duration is not None and last_end_time < total_duration):
+            delta = total_duration - segment_start
+
+            if (min_gap_length is None or delta >= min_gap_length):
+                result.append( { 'start': last_end_time, 'end': total_duration, 'gap': True } )
+
+        return result
+
     def adjust_whisper_timestamp(self, segments: Iterator[dict], adjust_seconds: float, max_source_time: float = None):
         result = []
 
@@ -178,11 +223,15 @@ class AbstractTranscription(ABC):
         return result
 
 class VadSileroTranscription(AbstractTranscription):
-    def __init__(self):
-        super().__init__(SEGMENT_PADDING_LEFT, SEGMENT_PADDING_RIGHT, MAX_SILENT_PERIOD)
-        self.model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad')
-
-        (self.get_speech_timestamps, _, _, _, _) = utils
+    def __init__(self, transcribe_non_speech: bool = False, copy = None):
+        super().__init__(SEGMENT_PADDING_LEFT, SEGMENT_PADDING_RIGHT, MAX_SILENT_PERIOD, transcribe_non_speech)
+
+        if copy:
+            self.model = copy.model
+            self.get_speech_timestamps = copy.get_speech_timestamps
+        else:
+            self.model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad')
+            (self.get_speech_timestamps, _, _, _, _) = utils
 
     def get_transcribe_timestamps(self, audio: str):
         wav = self.get_audio_segment(audio)