Add Meta-Llama-3-8B-Instruct ctranslate2 as the translation model to use.

app.py

@@ -921,6 +921,8 @@ class WhisperTranscriber:
         madlad400LangName:  str = dataDict.pop("madlad400LangName")
         seamlessModelName:  str = dataDict.pop("seamlessModelName")
         seamlessLangName:   str = dataDict.pop("seamlessLangName")
+        LlamaModelName:     str = dataDict.pop("LlamaModelName")
+        LlamaLangName:      str = dataDict.pop("LlamaLangName")
         
         translationBatchSize:         int  = dataDict.pop("translationBatchSize")
         translationNoRepeatNgramSize: int  = dataDict.pop("translationNoRepeatNgramSize")
@@ -954,6 +956,10 @@ class WhisperTranscriber:
             selectedModelName = seamlessModelName if seamlessModelName is not None and len(seamlessModelName) > 0 else "seamless-m4t-v2-large/facebook"
             selectedModel = next((modelConfig for modelConfig in self.app_config.models["seamless"] if modelConfig.name == selectedModelName), None)
             translationLang = get_lang_from_seamlessT_Tx_name(seamlessLangName)
+        elif translateInput == "Llama" and LlamaLangName is not None and len(LlamaLangName) > 0:
+            selectedModelName = LlamaModelName if LlamaModelName is not None and len(LlamaModelName) > 0 else "Meta-Llama-3-8B-Instruct-ct2-int8_float16/avan"
+            selectedModel = next((modelConfig for modelConfig in self.app_config.models["Llama"] if modelConfig.name == selectedModelName), None)
+            translationLang = get_lang_from_m2m100_name(LlamaLangName)
 
         if translationLang is not None:
             translationModel = TranslationModel(modelConfig=selectedModel, whisperLang=inputLang, translationLang=translationLang, batchSize=translationBatchSize, noRepeatNgramSize=translationNoRepeatNgramSize, numBeams=translationNumBeams, torchDtypeFloat16=translationTorchDtypeFloat16, usingBitsandbytes=translationUsingBitsandbytes)
@@ -1023,6 +1029,7 @@ def create_ui(app_config: ApplicationConfig):
     ALMA_models = app_config.get_model_names("ALMA")
     madlad400_models = app_config.get_model_names("madlad400")
     seamless_models = app_config.get_model_names("seamless")
+    Llama_models = app_config.get_model_names("Llama")
     if not torch.cuda.is_available(): # Loading only quantized or models with medium-low parameters in an environment without GPU support.
         nllb_models = list(filter(lambda nllb: any(name in nllb for name in ["-600M", "-1.3B", "-3.3B-ct2"]), nllb_models))
         m2m100_models = list(filter(lambda m2m100: "12B" not in m2m100, m2m100_models))
@@ -1057,20 +1064,24 @@ def create_ui(app_config: ApplicationConfig):
         gr.Dropdown(label="seamless - Model (for translate)", choices=seamless_models, elem_id="seamlessModelName"),
         gr.Dropdown(label="seamless - Language", choices=sorted(get_lang_seamlessT_Tx_names()), elem_id="seamlessLangName"),
     }
+    common_Llama_inputs = lambda : {
+        gr.Dropdown(label="Llama - Model (for translate)", choices=Llama_models, elem_id="LlamaModelName"),
+        gr.Dropdown(label="Llama - Language", choices=sorted(get_lang_m2m100_names()), elem_id="LlamaLangName"),
+    }
     
     common_translation_inputs = lambda : {
         gr.Number(label="Translation - Batch Size", precision=0, value=app_config.translation_batch_size, elem_id="translationBatchSize"),
-        gr.Number(label="Translation - No Repeat Ngram Size", precision=0, value=app_config.translation_no_repeat_ngram_size, elem_id="translationNoRepeatNgramSize"),
-        gr.Number(label="Translation - Num Beams", precision=0, value=app_config.translation_num_beams, elem_id="translationNumBeams"),
+        gr.Number(label="Translation - No Repeat Ngram Size", precision=0, value=app_config.translation_no_repeat_ngram_size, elem_id="translationNoRepeatNgramSize", info="Prevent repetitions of ngrams with this size (set 0 to disable)."),
+        gr.Number(label="Translation - Num Beams", precision=0, value=app_config.translation_num_beams, elem_id="translationNumBeams", info="Beam size (1 for greedy search)."),
         gr.Checkbox(label="Translation - Torch Dtype float16", visible=torch.cuda.is_available(), value=app_config.translation_torch_dtype_float16, info="Load the float32 translation model with float16 when the system supports GPU (reducing VRAM usage, not applicable to models that have already been quantized, such as Ctranslate2, GPTQ, GGUF)", elem_id="translationTorchDtypeFloat16"),
         gr.Radio(label="Translation - Using Bitsandbytes", visible=torch.cuda.is_available(), choices=[None, "int8", "int4"], value=app_config.translation_using_bitsandbytes, info="Load the float32 translation model into mixed-8bit or 4bit precision quantized model when the system supports GPU (reducing VRAM usage, not applicable to models that have already been quantized, such as Ctranslate2, GPTQ, GGUF)", elem_id="translationUsingBitsandbytes"),
     }
 
     common_vad_inputs = lambda : {
         gr.Dropdown(choices=["none", "silero-vad", "silero-vad-skip-gaps", "silero-vad-expand-into-gaps", "periodic-vad"], value=app_config.default_vad, label="VAD", elem_id="vad"),
-        gr.Number(label="VAD - Merge Window (s)", precision=0, value=app_config.vad_merge_window, elem_id="vadMergeWindow"),
-        gr.Number(label="VAD - Max Merge Size (s)", precision=0, value=app_config.vad_max_merge_size, elem_id="vadMaxMergeSize"),
-        gr.Number(label="VAD - Process Timeout (s)", precision=0, value=app_config.vad_process_timeout, elem_id="vadPocessTimeout"),
+        gr.Number(label="VAD - Merge Window (s)", precision=0, value=app_config.vad_merge_window, elem_id="vadMergeWindow", info="If set, any adjacent speech sections that are at most this number of seconds apart will be automatically merged."),
+        gr.Number(label="VAD - Max Merge Size (s)", precision=0, value=app_config.vad_max_merge_size, elem_id="vadMaxMergeSize", info="Disables merging of adjacent speech sections if they are this number of seconds long."),
+        gr.Number(label="VAD - Process Timeout (s)", precision=0, value=app_config.vad_process_timeout, elem_id="vadPocessTimeout", info="This configures the number of seconds until a process is killed due to inactivity, freeing RAM and video memory. The default value is 30 minutes."),
     }
     
     common_word_timestamps_inputs = lambda : {
@@ -1148,12 +1159,16 @@ def create_ui(app_config: ApplicationConfig):
                         with gr.Tab(label="seamless") as seamlessTab:
                             with gr.Row():
                                 inputDict.update(common_seamless_inputs())
+                        with gr.Tab(label="Llama") as llamaTab:
+                            with gr.Row():
+                                inputDict.update(common_Llama_inputs())
                         m2m100Tab.select(fn=lambda: "m2m100", inputs = [], outputs= [translateInput] )
                         nllbTab.select(fn=lambda: "nllb", inputs = [], outputs= [translateInput] )
                         mt5Tab.select(fn=lambda: "mt5", inputs = [], outputs= [translateInput] )
                         almaTab.select(fn=lambda: "ALMA", inputs = [], outputs= [translateInput] )
                         madlad400Tab.select(fn=lambda: "madlad400", inputs = [], outputs= [translateInput] )
                         seamlessTab.select(fn=lambda: "seamless", inputs = [], outputs= [translateInput] )
+                        llamaTab.select(fn=lambda: "Llama", inputs = [], outputs= [translateInput] )
                     with gr.Column():
                         with gr.Tab(label="URL") as UrlTab:
                             inputDict.update({gr.Text(label="URL (YouTube, etc.)", elem_id = "urlData")})
@@ -1164,14 +1179,14 @@ def create_ui(app_config: ApplicationConfig):
                         UrlTab.select(fn=lambda: "urlData", inputs = [], outputs= [sourceInput] )
                         UploadTab.select(fn=lambda: "multipleFiles", inputs = [], outputs= [sourceInput] )
                         MicTab.select(fn=lambda: "microphoneData", inputs = [], outputs= [sourceInput] )
-                        inputDict.update({gr.Dropdown(choices=["transcribe", "translate"], label="Task", value=app_config.task, elem_id = "task")})
+                        inputDict.update({gr.Dropdown(choices=["transcribe", "translate"], label="Task", value=app_config.task, elem_id = "task", info="Select the task - either \"transcribe\" to transcribe the audio to text, or \"translate\" to translate it to English.")})
                         with gr.Accordion("VAD options", open=False):
                             inputDict.update(common_vad_inputs())
                             if isFull:
                                 inputDict.update({
-                                    gr.Number(label="VAD - Padding (s)", precision=None, value=app_config.vad_padding, elem_id = "vadPadding"),
-                                    gr.Number(label="VAD - Prompt Window (s)", precision=None, value=app_config.vad_prompt_window, elem_id = "vadPromptWindow"),
-                                    gr.Dropdown(choices=VAD_INITIAL_PROMPT_MODE_VALUES, label="VAD - Initial Prompt Mode", value=app_config.vad_initial_prompt_mode, elem_id = "vadInitialPromptMode")})
+                                    gr.Number(label="VAD - Padding (s)", precision=None, value=app_config.vad_padding, elem_id = "vadPadding", info="The number of seconds (floating point) to add to the beginning and end of each speech section. Setting this to a number larger than zero ensures that Whisper is more likely to correctly transcribe a sentence in the beginning of a speech section. However, this also increases the probability of Whisper assigning the wrong timestamp to each transcribed line. The default value is 1 second."),
+                                    gr.Number(label="VAD - Prompt Window (s)", precision=None, value=app_config.vad_prompt_window, elem_id = "vadPromptWindow", info="The text of a detected line will be included as a prompt to the next speech section, if the speech section starts at most this number of seconds after the line has finished. For instance, if a line ends at 10:00, and the next speech section starts at 10:04, the line's text will be included if the prompt window is 4 seconds or more (10:04 - 10:00 = 4 seconds)."),
+                                    gr.Dropdown(choices=VAD_INITIAL_PROMPT_MODE_VALUES, label="VAD - Initial Prompt Mode", value=app_config.vad_initial_prompt_mode, elem_id = "vadInitialPromptMode", info="prepend_all_segments: prepend the initial prompt to each VAD segment, prepend_first_segment: just the first segment")})
                         with gr.Accordion("Word Timestamps options", open=False):
                             inputDict.update(common_word_timestamps_inputs())
                             if isFull:
@@ -1250,12 +1265,16 @@ def create_ui(app_config: ApplicationConfig):
                         with gr.Tab(label="seamless") as seamlessTab:
                             with gr.Row():
                                 inputDict.update(common_seamless_inputs())
+                        with gr.Tab(label="Llama") as llamaTab:
+                            with gr.Row():
+                                inputDict.update(common_Llama_inputs())
                         m2m100Tab.select(fn=lambda: "m2m100", inputs = [], outputs= [translateInput] )
                         nllbTab.select(fn=lambda: "nllb", inputs = [], outputs= [translateInput] )
                         mt5Tab.select(fn=lambda: "mt5", inputs = [], outputs= [translateInput] )
                         almaTab.select(fn=lambda: "ALMA", inputs = [], outputs= [translateInput] )
                         madlad400Tab.select(fn=lambda: "madlad400", inputs = [], outputs= [translateInput] )
                         seamlessTab.select(fn=lambda: "seamless", inputs = [], outputs= [translateInput] )
+                        llamaTab.select(fn=lambda: "Llama", inputs = [], outputs= [translateInput] )
                     with gr.Column():
                         inputDict.update({
                             gr.Dropdown(label="Input - Language", choices=sorted(get_lang_whisper_names()), value=app_config.language, elem_id="inputLangName"),

config.json5

@@ -292,6 +292,14 @@
         "url": "facebook/seamless-m4t-v2-large",
         "type": "huggingface"
       }
+    ],
+    "Llama": [
+      {
+        "name": "Meta-Llama-3-8B-Instruct-ct2-int8_float16/avan",
+        "url": "avans06/Meta-Llama-3-8B-Instruct-ct2-int8_float16",
+        "type": "huggingface",
+        "tokenizer_url": "avans06/Meta-Llama-3-8B-Instruct-ct2-int8_float16"
+      }
     ]
   },
   // Configuration options that will be used if they are not specified in the command line arguments.

src/config.py

@@ -50,7 +50,7 @@ class VadInitialPromptMode(Enum):
             return None
 
 class ApplicationConfig:
-    def __init__(self, models: Dict[Literal["whisper", "m2m100", "nllb", "mt5", "ALMA", "madlad400", "seamless"], List[ModelConfig]],
+    def __init__(self, models: Dict[Literal["whisper", "m2m100", "nllb", "mt5", "ALMA", "madlad400", "seamless", "Llama"], List[ModelConfig]],
                  input_audio_max_duration: int = 600, share: bool = False, server_name: str = None, server_port: int = 7860, 
                  queue_concurrency_count: int = 1, delete_uploaded_files: bool = True,
                  whisper_implementation: str = "whisper", default_model_name: str = "medium", 
@@ -185,7 +185,7 @@ class ApplicationConfig:
             # Load using json5
             data = json5.load(f)
             data_models = data.pop("models", [])
-            models: Dict[Literal["whisper", "m2m100", "nllb", "mt5", "ALMA", "madlad400", "seamless"], List[ModelConfig]] = {
+            models: Dict[Literal["whisper", "m2m100", "nllb", "mt5", "ALMA", "madlad400", "seamless", "Llama"], List[ModelConfig]] = {
                 key: [ModelConfig(**item) for item in value]
                 for key, value in data_models.items()
             }

src/translation/translationModel.py

@@ -27,7 +27,7 @@ class TranslationModel:
         localFilesOnly: bool = False,
         loadModel: bool = False,
     ):
-        """Initializes the M2M100 / Nllb-200 / mt5 / ALMA / madlad400 / seamless-m4t translation model.
+        """Initializes the M2M100 / Nllb-200 / mt5 / ALMA / madlad400 / seamless-m4t / Llama translation model.
 
         Args:
           modelConfig: Config of the model to use (distilled-600M, distilled-1.3B, 
@@ -230,6 +230,9 @@ class TranslationModel:
                 if "ALMA" in self.modelPath:
                     self.ALMAPrefix = "Translate this from " + self.whisperLang.whisper.names[0] + " to " + self.translationLang.whisper.names[0] + ":\n" + self.whisperLang.whisper.names[0] + ": "
                     self.transModel = ctranslate2.Generator(**kwargsModel)
+                elif "Llama" in self.modelPath:
+                    self.roleSystem = {"role": "system", "content":"You are an excellent and professional translation master who understands languages from all around the world. Please directly translate the following sentence from " + self.whisperLang.whisper.names[0] + " to " + self.translationLang.whisper.names[0] + ", please simply provide the translation below without further explanation and without using any emojis."}
+                    self.transModel = ctranslate2.Generator(**kwargsModel)
                 else:
                     if "nllb" in self.modelPath:
                         kwargsTokenizer.update({"src_lang": self.whisperLang.nllb.code})
@@ -243,6 +246,8 @@ class TranslationModel:
                 self.transTokenizer = transformers.AutoTokenizer.from_pretrained(**kwargsTokenizer)
                 if "m2m100" in self.modelPath:
                     self.targetPrefix = [self.transTokenizer.lang_code_to_token[self.translationLang.m2m100.code]]
+                elif "Llama" in self.modelPath:
+                    self.terminators = [self.transTokenizer.eos_token_id, self.transTokenizer.convert_tokens_to_ids("<|eot_id|>")]
             elif "mt5" in self.modelPath:
                 self.mt5Prefix = self.whisperLang.whisper.code + "2" + self.translationLang.whisper.code + ": "
                 kwargsTokenizer.update({"pretrained_model_name_or_path": self.modelPath, "legacy": False})
@@ -382,6 +387,12 @@ class TranslationModel:
                     output = self.transModel.generate_batch([source], max_length=max_length, max_batch_size=self.batchSize, no_repeat_ngram_size=self.noRepeatNgramSize, beam_size=self.numBeams, sampling_temperature=0.7, sampling_topp=0.9, repetition_penalty=1.1, include_prompt_in_result=False) #, sampling_topk=40
                     target = output[0]
                     result = self.transTokenizer.decode(target.sequences_ids[0])
+                elif "Llama" in self.modelPath:
+                    input_ids = self.transTokenizer.apply_chat_template([self.roleSystem, {"role": "user", "content": "'" + text + "', \n" + self.translationLang.whisper.names[0] + ":"}], tokenize=False, add_generation_prompt=True)
+                    source = self.transTokenizer.convert_ids_to_tokens(self.transTokenizer.encode(input_ids))
+                    output = self.transModel.generate_batch([source], max_length=max_length, max_batch_size=self.batchSize, no_repeat_ngram_size=self.noRepeatNgramSize, beam_size=self.numBeams, sampling_temperature=0.7, sampling_topp=0.9, include_prompt_in_result=False, end_token=self.terminators)
+                    target = output[0]
+                    result = self.transTokenizer.decode(target.sequences_ids[0])
                 elif "madlad400" in self.modelPath:
                     source = self.transTokenizer.convert_ids_to_tokens(self.transTokenizer.encode(self.madlad400Prefix + text))
                     output = self.transModel.translate_batch([source], max_batch_size=self.batchSize, no_repeat_ngram_size=self.noRepeatNgramSize, beam_size=self.numBeams)
@@ -424,7 +435,8 @@ _MODELS = ["nllb-200",
            "mt5",
            "ALMA",
            "madlad400",
-           "seamless"]
+           "seamless"
+           "Llama"]
 
 def check_model_name(name):
     return any(allowed_name in name for allowed_name in _MODELS)