[update]add sent_tokenize

main.py

@@ -10,6 +10,7 @@ hf_hub_cache = (project_path / "cache/huggingface/hub").as_posix()
 os.environ["HUGGINGFACE_HUB_CACHE"] = hf_hub_cache
 
 import gradio as gr
+import nltk
 from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
 
 
@@ -41,13 +42,19 @@ def main():
         tokenizer = model_group["tokenizer"]
 
         tokenizer.src_lang = src_lang
-        encoded_src = tokenizer(src_text, return_tensors="pt")
-        generated_tokens = model.generate(**encoded_src,
-                                          forced_bos_token_id=tokenizer.get_lang_id(tgt_lang),
-                                          )
-        result = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
 
-        return result[0]
+        src_t_list = nltk.sent_tokenize(src_text)
+
+        result = ""
+        for src_t in src_t_list:
+            encoded_src = tokenizer(src_t, return_tensors="pt")
+            generated_tokens = model.generate(**encoded_src,
+                                              forced_bos_token_id=tokenizer.get_lang_id(tgt_lang),
+                                              )
+            text_decoded = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
+            result += text_decoded[0]
+
+        return result
 
     title = "Multilingual Machine Translation"
 

requirements.txt

@@ -2,3 +2,4 @@ gradio==3.20.1
 transformers==4.30.2
 torch==1.13.1
 sentencepiece==0.1.99
+nltk==3.8.1