Update app.py

app.py

@@ -28,8 +28,10 @@ SPEAKER_KRC_TTS = 'b_krc'
 REPO_TTS_PATH = "snakers4/silero-models"
 MODEL_TTS_PATH = "silero_tts"
 
-LANGUAGE = pd.DataFrame({"language": ["Къарачай-Малкъар тил", "Русский язык"], "token": ["krc_Cyrl", "rus_Cyrl"]})
-DIALECT = pd.DataFrame({"dialect": ["дж\ч", "ж\ч", "з\ц"], "short_name": ["qrc", "hlm", "mqr"]})
+# LANGUAGE = pd.DataFrame({"language": ["Къарачай-Малкъар тил", "Русский язык"], "token": ["krc_Cyrl", "rus_Cyrl"]})
+LANGUAGE = {"Къарачай-Малкъар тил": "krc_Cyrl", "Русский язык": "rus_Cyrl"}
+# DIALECT = pd.DataFrame({"dialect": ["дж\ч", "ж\ч", "з\ц"], "short_name": ["qrc", "hlm", "mqr"]})
+DIALECT = {"дж\ч": "qrc", "ж\ч": "hlm", "з\ц": "mqr"}
 TYPE = pd.DataFrame({"krc": ["Кёчюрюўчю", "Сёзлюк", "Сёлешиўчю"], "rus": ["Переводчик", "Словарь", "Озвучка"], "eng": ["Translator", "Dictionary", "Voice"], "tur": ["Çevirmen", "Sözlük", "Seslendirme"], "short_name": ["translator", "dictionary", "tts"]})
 
 SYSTEM_LANG = "rus"
@@ -42,7 +44,7 @@ NAMES = pd.DataFrame({
 })
 
 
-DEVICE = 'gpu' if torch.cuda.is_available() else 'cpu'
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
 
 device = torch.device(DEVICE)
 
@@ -73,7 +75,7 @@ model_tts, _  = torch.hub.load(repo_or_dir = REPO_TTS_PATH,
 model_tts.to(device)
 
 # 4. Fix tokenizer
-#def fixTokenizer(tokenizer, new_lang='krc_Cyrl'):
+# def fixTokenizer(tokenizer, new_lang='krc_Cyrl'):
 #    """
 #    Add a new language token to the tokenizer vocabulary
 #    (this should be done each time after its initialization)
@@ -94,10 +96,200 @@ model_tts.to(device)
 
 #fixTokenizer(tokenizer)
 
-# 5. Change letters
 
-def fromModel(str, dialect = "qrc"):
-  if dialect == "qrc":
+class Translator:
+  """
+    Class for translator NLLB-200.
+
+    Параметры:
+    - model:     Модель
+    - tokenizer: Токенизатор
+
+    Функция translate алады:
+    - text (str):             Текст
+    - src_lang (str):         Тебреген тил
+    - tgt_lang (str):         Тил таба
+    - dialect (int): Диалект
+
+    Чыгъарады:
+    - translated (str): Кёчюрюлгени
+
+    """
+  def __init__(self, tokenizer, model) -> None:
+    self.model = model
+    self.tokenizer = tokenizer
+  # Change letters
+  def _fromModel(self, str: str, dialect: str = "qrc") -> str:
+    if dialect == "qrc":
+        str = str.replace("тюйюл", "тюл") 
+        str = str.replace("Тюйюл", "Тюл") 
+        str = str.replace("уку", "гылын qуш") 
+        str = str.replace("Уку", "Гылын qуш") 
+        str = str.replace("хораз", "гугурукку") 
+        str = str.replace("Хораз", "Гугурукку") 
+        str = str.replace("юзмез", "qум") 
+        str = str.replace("Юзмез", "Qум") 
+        str = str.replace("jиля", "jыла") 
+        str = str.replace("Jиля", "Jыла") 
+        str = str.replace("ярабий", "арабин") 
+        str = str.replace("арабий", "арабин") 
+        str = str.replace("Ярабий", "Арабин") 
+        str = str.replace("Арабий", "Арабин") 
+        str = str.replace("нтта", "нтда") 
+        str = str.replace("ртте", "ртде") 
+        str = str.replace("jамауат", "jамаgат") 
+        str = str.replace("jамаwат", "jамаgат") 
+        str = str.replace("Jамауат", "Jамаgат") 
+        str = str.replace("Jамаwат", "Jамаgат") 
+        str = str.replace("шуёх", "шох") 
+        str = str.replace("Шуёх", "Шох") 
+        str = str.replace("шёндю", "бусаgат") 
+        str = str.replace("Шёндю", "Бусаgат") 
+        str = str.replace("уgай", "оgай") 
+        str = str.replace("Уgай", "Оgай") 
+        # str = str.replace("терк", "тез") 
+        str = str.replace("саnа", "сенnе") 
+        str = str.replace("сеnе", "сенnе") 
+        str = str.replace("Саnа", "Сенnе") 
+        str = str.replace("Сеnе", "Сенnе") 
+        str = str.replace("маnа", "менnе") 
+        str = str.replace("меnе", "менnе") 
+        str = str.replace("Маnа", "Менnе") 
+        str = str.replace("Меnе", "Менnе") 
+        str = str.replace("аяq jол", "jахтана") 
+        str = str.replace("Аяq jол", "Jахтана") 
+        str = str.replace("сыbат", "сыфат") 
+        str = str.replace("Сыbат", "Сыфат") 
+        str = str.replace("b", "б") 
+        str = str.replace("q", "къ") 
+        str = str.replace("Q", "Къ") 
+        str = str.replace("g", "гъ") 
+        str = str.replace("G", "Гъ") 
+        str = str.replace("j", "дж") 
+        str = str.replace("J", "Дж") 
+        str = str.replace("w", "ў") 
+        str = str.replace("W", "Ў") 
+        str = str.replace("n", "нг") 
+        str = str.replace("N", "Нг")
+    elif dialect == "hlm":
+        str = str.replace("тюл", "тюйюл") 
+        str = str.replace("Тюл", "Тюйюл") 
+        str = str.replace("гылын qуш", "уку") 
+        str = str.replace("Гылын qуш", "Уку") 
+        str = str.replace("гугурукку", "хораз") 
+        str = str.replace("Гугурукку", "Хораз") 
+        str = str.replace("qум", "юзмез") 
+        str = str.replace("Qум", "Юзмез") 
+        str = str.replace("jыла", "jиля") 
+        str = str.replace("Jыла", "Jиля") 
+        str = str.replace("арабин", "ярабий") 
+        str = str.replace("арабий", "ярабий") 
+        str = str.replace("Арабин", "Ярабий") 
+        str = str.replace("Арабий", "Ярабий") 
+        str = str.replace("нтда", "нтта") 
+        str = str.replace("ртде", "ртте") 
+        str = str.replace("jамаgат", "jамаwат") 
+        str = str.replace("Jамаgат", "Jамаwат") 
+        str = str.replace("шох", "шуёх") 
+        str = str.replace("Шох", "Шуёх") 
+        str = str.replace("бусаgат", "шёндю") 
+        str = str.replace("Бусаgат", "Шёндю") 
+        str = str.replace("оgай", "уgай") 
+        str = str.replace("Оgай", "Уgай") 
+        str = str.replace("тез", "терк") 
+        str = str.replace("сенnе", "саnа") 
+        str = str.replace("сеnе", "саnа") 
+        str = str.replace("Сенnе", "Саnа") 
+        str = str.replace("Сеnе", "Саnа") 
+        str = str.replace("менnе", "маnа") 
+        str = str.replace("меnе", "маnа") 
+        str = str.replace("Менnе", "Маnа") 
+        str = str.replace("Меnе", "Маnа") 
+        str = str.replace("jахтана", "аяq jол") 
+        str = str.replace("Jахтана", "аяq jол") 
+        str = str.replace("хо", "хаw") 
+        str = str.replace("Хо", "Хаw") 
+        str = str.replace("сыbат", "сыфат") 
+        str = str.replace("Сыbат", "Сыфат") 
+        str = str.replace("b", "п") 
+        str = str.replace("q", "къ") 
+        str = str.replace("Q", "Къ") 
+        str = str.replace("g", "гъ") 
+        str = str.replace("G", "Гъ") 
+        str = str.replace("j", "ж") 
+        str = str.replace("J", "Ж") 
+        str = str.replace("w", "ў") 
+        str = str.replace("W", "Ў") 
+        str = str.replace("n", "нг") 
+        str = str.replace("N", "Нг")
+    elif dialect == "mqr":
+        str = str.replace("тюл", "тюйюл") 
+        str = str.replace("Тюл", "Тюйюл") 
+        str = str.replace("гылын qуш", "уку") 
+        str = str.replace("Гылын qуш", "Уку") 
+        str = str.replace("гугурукку", "хораз") 
+        str = str.replace("Гугурукку", "Хораз") 
+        str = str.replace("qум", "юзмез") 
+        str = str.replace("Qум", "Юзмез") 
+        str = str.replace("jыла", "jиля") 
+        str = str.replace("Jыла", "Jиля") 
+        str = str.replace("арабин", "ярабий") 
+        str = str.replace("арабий", "ярабий") 
+        str = str.replace("Арабин", "Ярабий") 
+        str = str.replace("Арабий", "Ярабий") 
+        str = str.replace("нтда", "нтта") 
+        str = str.replace("ртде", "ртте") 
+        str = str.replace("jамаgат", "жамаwат") 
+        str = str.replace("Jамаgат", "Жамаwат") 
+        str = str.replace("шох", "шуёх") 
+        str = str.replace("Шох", "Шуёх") 
+        str = str.replace("бусаgат", "шёндю") 
+        str = str.replace("Бусаgат", "Шёндю") 
+        str = str.replace("оgай", "уgай") 
+        str = str.replace("Оgай", "Уgай") 
+        str = str.replace("тез", "терк") 
+        str = str.replace("сенnе", "саnа") 
+        str = str.replace("сеnе", "саnа") 
+        str = str.replace("Сенnе", "Саnа") 
+        str = str.replace("Сеnе", "Саnа") 
+        str = str.replace("менnе", "маnа") 
+        str = str.replace("меnе", "маnа") 
+        str = str.replace("Менnе", "Маnа") 
+        str = str.replace("Меnе", "Маnа") 
+        str = str.replace("jахтана", "аяq jол") 
+        str = str.replace("Jахтана", "аяq jол") 
+        str = str.replace("хо", "хаw") 
+        str = str.replace("Хо", "Хаw") 
+        str = str.replace("сыbат", "сыфат") 
+        str = str.replace("Сыbат", "Сыфат") 
+        str = str.replace("b", "п") 
+        str = str.replace("q", "къ") 
+        str = str.replace("Q", "Къ") 
+        str = str.replace("g", "гъ") 
+        str = str.replace("G", "Гъ") 
+        str = str.replace("j", "з") 
+        str = str.replace("J", "З") 
+        str = str.replace("w", "ў") 
+        str = str.replace("W", "Ў") 
+        str = str.replace("n", "нг") 
+        str = str.replace("N", "Нг") 
+        str = str.replace("ч", "ц") 
+        str = str.replace("Ч", "Ц") 
+        str = str.replace("п", "ф") 
+        str = str.replace("П", "Ф") 
+        str = str.replace("къ|гъ", "х")
+    return str
+      
+  def _toModel(self, str: str) -> str:
+      str = str.replace("дж", "j") 
+      str = str.replace("Дж", "J") 
+      str = str.replace("ДЖ", "J") 
+      str = str.replace("ж", "j") 
+      str = str.replace("Ж", "J")
+      str = str.replace("себеп", "себеb") 
+      str = str.replace("себеб", "себеb") 
+      str = str.replace("Себеп", "Себеb") 
+      str = str.replace("Себеб", "Себеb") 
       str = str.replace("тюйюл", "тюл") 
       str = str.replace("Тюйюл", "Тюл") 
       str = str.replace("уку", "гылын qуш") 
@@ -106,346 +298,252 @@ def fromModel(str, dialect = "qrc"):
       str = str.replace("Хораз", "Гугурукку") 
       str = str.replace("юзмез", "qум") 
       str = str.replace("Юзмез", "Qум") 
+      str = str.replace("арап", "араb") 
+      str = str.replace("араб", "араb") 
+      str = str.replace("Арап", "Араb")
+      str = str.replace("Араб", "Араb")
       str = str.replace("jиля", "jыла") 
+      str = str.replace("jыла", "jыла") 
+      str = str.replace("jыла", "jыла") 
       str = str.replace("Jиля", "Jыла") 
+      str = str.replace("Jыла", "Jыла") 
+      str = str.replace("Jыла", "Jыла") 
       str = str.replace("ярабий", "арабин") 
       str = str.replace("арабий", "арабин") 
       str = str.replace("Ярабий", "Арабин") 
       str = str.replace("Арабий", "Арабин") 
       str = str.replace("нтта", "нтда") 
       str = str.replace("ртте", "ртде") 
+      str = str.replace("jамагъат", "jамаgат") 
       str = str.replace("jамауат", "jамаgат") 
-      str = str.replace("jамаwат", "jамаgат") 
+      str = str.replace("jамагъат", "jамаgат")
+      str = str.replace("jамауат", "jамаgат")
+      str = str.replace("Jамагъат", "Jамаgат") 
       str = str.replace("Jамауат", "Jамаgат") 
-      str = str.replace("Jамаwат", "Jамаgат") 
+      str = str.replace("Jамагъат", "Jамаgат") 
+      str = str.replace("Jамаўат", "Jамаgат") 
       str = str.replace("шуёх", "шох") 
       str = str.replace("Шуёх", "Шох") 
       str = str.replace("шёндю", "бусаgат") 
+      str = str.replace("бусагъат", "бусаgат") 
       str = str.replace("Шёндю", "Бусаgат") 
-      str = str.replace("уgай", "оgай") 
-      str = str.replace("Уgай", "Оgай") 
+      str = str.replace("Бусагъат", "Бусаgат") 
+      str = str.replace("угъай", "оgай")
+      str = str.replace("огъай", "оgай")
+      str = str.replace("Угъай", "Оgай") 
+      str = str.replace("Огъай", "Оgай") 
+      # str = str.replace("терк", "тез") 
       # str = str.replace("терк", "тез") 
-      str = str.replace("саnа", "сенnе") 
-      str = str.replace("сеnе", "сенnе") 
-      str = str.replace("Саnа", "Сенnе") 
-      str = str.replace("Сеnе", "Сенnе") 
-      str = str.replace("маnа", "менnе") 
-      str = str.replace("меnе", "менnе") 
-      str = str.replace("Маnа", "Менnе") 
-      str = str.replace("Меnе", "Менnе") 
-      str = str.replace("аяq jол", "jахтана") 
-      str = str.replace("Аяq jол", "Jахтана") 
-      str = str.replace("сыbат", "сыфат") 
-      str = str.replace("Сыbат", "Сыфат") 
-      str = str.replace("b", "б") 
-      str = str.replace("q", "къ") 
-      str = str.replace("Q", "Къ") 
-      str = str.replace("g", "гъ") 
-      str = str.replace("G", "Гъ") 
-      str = str.replace("j", "дж") 
-      str = str.replace("J", "Дж") 
-      str = str.replace("w", "ў") 
-      str = str.replace("W", "Ў") 
-      str = str.replace("n", "нг") 
-      str = str.replace("N", "Нг")
-  elif dialect == "hlm":
-      str = str.replace("тюл", "тюйюл") 
-      str = str.replace("Тюл", "Тюйюл") 
-      str = str.replace("гылын qуш", "уку") 
-      str = str.replace("Гылын qуш", "Уку") 
-      str = str.replace("гугурукку", "хораз") 
-      str = str.replace("Гугурукку", "Хораз") 
-      str = str.replace("qум", "юзмез") 
-      str = str.replace("Qум", "Юзмез") 
-      str = str.replace("jыла", "jиля") 
-      str = str.replace("Jыла", "Jиля") 
-      str = str.replace("арабин", "ярабий") 
-      str = str.replace("арабий", "ярабий") 
-      str = str.replace("Арабин", "Ярабий") 
-      str = str.replace("Арабий", "Ярабий") 
-      str = str.replace("нтда", "нтта") 
-      str = str.replace("ртде", "ртте") 
-      str = str.replace("jамаgат", "jамаwат") 
-      str = str.replace("Jамаgат", "Jамаwат") 
-      str = str.replace("шох", "шуёх") 
-      str = str.replace("Шох", "Шуёх") 
-      str = str.replace("бусаgат", "шёндю") 
-      str = str.replace("Бусаgат", "Шёндю") 
-      str = str.replace("оgай", "уgай") 
-      str = str.replace("Оgай", "Уgай") 
-      str = str.replace("тез", "терк") 
-      str = str.replace("сенnе", "саnа") 
-      str = str.replace("сеnе", "саnа") 
-      str = str.replace("Сенnе", "Саnа") 
-      str = str.replace("Сеnе", "Саnа") 
-      str = str.replace("менnе", "маnа") 
-      str = str.replace("меnе", "маnа") 
-      str = str.replace("Менnе", "Маnа") 
-      str = str.replace("Меnе", "Маnа") 
-      str = str.replace("jахтана", "аяq jол") 
-      str = str.replace("Jахтана", "аяq jол") 
-      str = str.replace("хо", "хаw") 
-      str = str.replace("Хо", "Хаw") 
-      str = str.replace("сыbат", "сыфат") 
-      str = str.replace("Сыbат", "Сыфат") 
-      str = str.replace("b", "п") 
-      str = str.replace("q", "къ") 
-      str = str.replace("Q", "Къ") 
-      str = str.replace("g", "гъ") 
-      str = str.replace("G", "Гъ") 
-      str = str.replace("j", "ж") 
-      str = str.replace("J", "Ж") 
-      str = str.replace("w", "ў") 
-      str = str.replace("W", "Ў") 
-      str = str.replace("n", "нг") 
-      str = str.replace("N", "Нг")
-  elif dialect == "mqr":
-      str = str.replace("тюл", "тюйюл") 
-      str = str.replace("Тюл", "Тюйюл") 
-      str = str.replace("гылын qуш", "уку") 
-      str = str.replace("Гылын qуш", "Уку") 
-      str = str.replace("гугурукку", "хораз") 
-      str = str.replace("Гугурукку", "Хораз") 
-      str = str.replace("qум", "юзмез") 
-      str = str.replace("Qум", "Юзмез") 
-      str = str.replace("jыла", "jиля") 
-      str = str.replace("Jыла", "Jиля") 
-      str = str.replace("арабин", "ярабий") 
-      str = str.replace("арабий", "ярабий") 
-      str = str.replace("Арабин", "Ярабий") 
-      str = str.replace("Арабий", "Ярабий") 
-      str = str.replace("нтда", "нтта") 
-      str = str.replace("ртде", "ртте") 
-      str = str.replace("jамаgат", "жамаwат") 
-      str = str.replace("Jамаgат", "Жамаwат") 
-      str = str.replace("шох", "шуёх") 
-      str = str.replace("Шох", "Шуёх") 
-      str = str.replace("бусаgат", "шёндю") 
-      str = str.replace("Бусаgат", "Шёндю") 
-      str = str.replace("оgай", "уgай") 
-      str = str.replace("Оgай", "Уgай") 
-      str = str.replace("тез", "терк") 
-      str = str.replace("сенnе", "саnа") 
-      str = str.replace("сеnе", "саnа") 
-      str = str.replace("Сенnе", "Саnа") 
-      str = str.replace("Сеnе", "Саnа") 
-      str = str.replace("менnе", "маnа") 
-      str = str.replace("меnе", "маnа") 
-      str = str.replace("Менnе", "Маnа") 
-      str = str.replace("Меnе", "Маnа") 
-      str = str.replace("jахтана", "аяq jол") 
-      str = str.replace("Jахтана", "аяq jол") 
-      str = str.replace("хо", "хаw") 
-      str = str.replace("Хо", "Хаw") 
-      str = str.replace("сыbат", "сыфат") 
-      str = str.replace("Сыbат", "Сыфат") 
-      str = str.replace("b", "п") 
-      str = str.replace("q", "къ") 
-      str = str.replace("Q", "Къ") 
-      str = str.replace("g", "гъ") 
-      str = str.replace("G", "Гъ") 
-      str = str.replace("j", "з") 
-      str = str.replace("J", "З") 
-      str = str.replace("w", "ў") 
-      str = str.replace("W", "Ў") 
-      str = str.replace("n", "нг") 
-      str = str.replace("N", "Нг") 
-      str = str.replace("ч", "ц") 
-      str = str.replace("Ч", "Ц") 
-      str = str.replace("п", "ф") 
-      str = str.replace("П", "Ф") 
-      str = str.replace("къ|гъ", "х")
-  return str
+      str = str.replace("санга", "сенnе") 
+      str = str.replace("сенге", "сенnе") 
+      str = str.replace("сеннге", "сенnе") 
+      str = str.replace("Санга", "Сенnе") 
+      str = str.replace("Сеннге", "Сенnе") 
+      str = str.replace("Сенге", "Сенnе") 
+      str = str.replace("манга", "менnе") 
+      str = str.replace("меннге", "менnе") 
+      str = str.replace("менге", "менnе") 
+      str = str.replace("Манга", "Менnе") 
+      str = str.replace("Меннге", "Менnе") 
+      str = str.replace("Менге", "Менnе") 
+      str = str.replace("аякъ jол", "jахтана") 
+      str = str.replace("аякъ jол", "jахтана") 
+      str = str.replace("jахтана", "jахтана") 
+      str = str.replace("jахтана", "jахтана") 
+      str = str.replace("Аякъ jол", "Jахтана") 
+      str = str.replace("Аякъ jол", "Jахтана") 
+      str = str.replace("Jахтана", "Jахтана") 
+      str = str.replace("Jахтана", "Jахтана") 
+      str = str.replace("къамж", "qамыzh") 
+      str = str.replace("къамыж", "qамыzh") 
+      str = str.replace("Къамж", "Qамыzh") 
+      str = str.replace("Къамыж", "Qамыzh") 
+      str = str.replace("къымыж", "qымыzh") 
+      str = str.replace("къымыж", "qымыzh") 
+      str = str.replace("Къымыж", "Qымыzh") 
+      str = str.replace("Къымыж", "Qымыzh") 
+      str = str.replace("хау", "хо") 
+      str = str.replace("хаў", "хо") 
+      str = str.replace("Хау", "Хо") 
+      str = str.replace("Хаў", "Хо") 
+      str = str.replace("уа", "wa") 
+      str = str.replace("ўа", "wa") 
+      str = str.replace("Уа", "Wa") 
+      str = str.replace("Ўа", "Wa") 
+      str = str.replace("п", "b") 
+      str = str.replace("б", "b") 
+      str = str.replace("къ", "q") 
+      str = str.replace("Къ", "Q") 
+      str = str.replace("КЪ", "Q") 
+      str = str.replace("гъ", "g") 
+      str = str.replace("Гъ", "G") 
+      str = str.replace("ГЪ", "G") 
+      str = str.replace("ц", "ч") 
+      str = str.replace("Ц", "Ч") 
+      str = str.replace("ф", "п") 
+      str = str.replace("сыпат", "сыфат") 
+      str = str.replace("Сыпат", "Сыфат") 
+      str = str.replace("Ф", "П") 
+      str = str.replace("(?<=[аыоуэеиёюя])у(?=[аыоуэеиёюя])|(?<=[аыоуэеиёюя])ў(?=[аыоуэеиёюя])|(?<=[АЫОУЭЕИЁЮЯ])у(?=[АЫОУЭЕИЁЮЯ])|(?<=[АЫОУЭЕИЁЮЯ])ў(?=[АЫОУЭЕИЁЮЯ])", "w")
+      str = str.replace("(?<=[аыоуэеиёюя])у|(?<=[аыоуэеиёюя])ў|(?<=[АЫОУЭЕИЁЮЯ])у|(?<=[АЫОУЭЕИЁЮЯ])ў", "w")
+      # str = str.replace("у(?=[аыоуэеиёюя])|ў(?=[аыоуэеиёюя])|у(?=[АЫОУЭЕИЁЮЯ])|ў(?=[АЫОУЭЕИЁЮЯ])", "w") 
+      # str = str.replace("У(?=[аыоуэеиёюя])|Ў(?=[аыоуэеиёюя])|У(?=[АЫОУЭЕИЁЮЯ])|Ў(?=[АЫОУЭЕИЁЮЯ])", "W") 
+      str = str.replace("zh", "ж") 
+      str = str.replace("нг", "n") 
+      str = str.replace("Нг", "  N")
+      str = str.replace("НГ", "  N")
+      return str
+
+  # structure
+  def _prepareTextAndStructure(self, text: str) -> tuple:  
+    """
+    The input text is divided into sentences, while maintaining the structure
+    """
+
+    # Разбиваем текст на предложения, сохраняя знаки препинания
+    #   .+?[.!?।ฯ؟](?:\s|$): Захватывает предложения, которые заканчиваются
+    # точкой, восклицательным или вопросительным знаком.
+
+    # |.+?(?:\n|$): Добавляет поддержку для разрыва строки (\n) или конца текста ($),
+    # если предложение не заканчивается знаком препинания.
+    segments = re.findall(pattern=r".+?[.!?।ฯ؟](?:\s|$)|.*?(?:\n|$)", string=text)
+
+    # Если последний элемент пустой, то его удаляем
+    if not segments[-1]:
+        segments = segments[:-1]
+
+    # Склеиваем разорванные предложения
+    merged_segments = []
+    buffer = ""
+
+    for i, segment in enumerate(segments):
+        # Проверяем, заканчивается ли текущий сегмент на .!? или пуст
+        if buffer:
+            buffer += " " + segment
+        else:
+            buffer = segment
+
+        # Если сегмент не заканчивается на .!? и следующий начинается с маленькой буквы
+        if (  # noqa: R507
+            not re.search(pattern=r"[.!?।ฯ؟](?:\s|$)", string=segment)  # noqa: ECE001
+            and i + 1 < len(segments)
+            and segments[i + 1].strip()
+            and ((segments[i + 1].strip()[0].islower()) or (segments[i + 1].strip()[0] in ["'", '"']))
+        ):
+            continue  # Склеиваем с следующим сегментом
+        else:
+            merged_segments.append(buffer)
+            buffer = ""
+
+    # Удаляем пустые сегменты и сохраняем пробелы
+    original_structure = []
+    for segment in merged_segments:
+        match = re.match(pattern=r"^(\s*)(.*?)(\s*)$", string=segment, flags=re.DOTALL)
+        if match:
+            original_structure.append((match.group(1), match.group(2), match.group(3)))
+
+    # Токенизируем только текстовые части сегментов
+    texts_to_translate = [seg[1] for seg in original_structure if seg[1].strip()]
+
+    return texts_to_translate, original_structure
+
+  def _recoverTranslatedToStructure(self, translated_texts: str, original_structure: list) -> str:
+    """
+    Translated sentences are embedded in the structure of the original text
+    """
+    # Восстанавливаем исходную структуру текста
+    translated_segments = []
+    translated_index = 0
+    for seg in original_structure:
+        if seg[1].strip():  # Если сегмент был переведён
+            translated_segments.append(f"{seg[0]}{translated_texts[translated_index]}{seg[2]}")
+            translated_index += 1
+        else:  # Если сегмент был пустым, оставляем его как есть
+            translated_segments.append(f"{seg[0]}{seg[1]}{seg[2]}")
+
+    return "".join(translated_segments)
+
+
+  # Translate function
+  def _translate(self, text: list | str, src_lang: str = 'rus_Cyrl', tgt_lang: str = 'krc_Cyrl',
+      a: int = 32, b: int = 3, max_input_length: int = 1024, num_beams: int = 3, **kwargs
+  ) -> list:
+      """Turn a text or a list of texts into a list of translations"""
+      self.tokenizer.src_lang = src_lang
+      self.tokenizer.tgt_lang = tgt_lang
+      inputs = self.tokenizer(
+          text, return_tensors='pt', padding=True, truncation=True,
+          max_length=max_input_length
+      )
+      #print(f'Inputs: {inputs}')
+      result = self.model.generate(
+          **inputs.to(self.model.device),
+          forced_bos_token_id=self.tokenizer.convert_tokens_to_ids(tgt_lang),
+          max_new_tokens=int(a + b * inputs.input_ids.shape[1]),
+          num_beams=num_beams, **kwargs
+      )
+      #print(f'Outputs: {result}')
+      return self.tokenizer.batch_decode(result, skip_special_tokens=True)
+
+  def translate(self, text: str, src_lang: str | None = None, tgt_lang: str | None = None, dialect: str | None = None) -> str:
+    # print(src_lang)
+    # print(trg_lang)
+    # print(dialect)
+    if dialect == "" or dialect is None:
+      # dialect = DIALECT.dialect[0] # "дж\ч"
+      dialect = list(DIALECT.keys())[0] # "дж\ч"
+    if src_lang == "" or src_lang is None:
+      # src_lang = LANGUAGE.language[1] # "Русский язык"
+      src_lang = list(LANGUAGE.keys())[1] # "Русский язык"
+    if tgt_lang == "" or tgt_lang is None:
+      # tgt_lang = LANGUAGE.language[0] # "Къарачай-Малкъар тил"
+      tgt_lang = list(LANGUAGE.keys())[0] # "Къарачай-Малкъар тил"
+      
+    # src_lang = "".join(LANGUAGE[LANGUAGE.language == src_lang].token.to_list())
+    # tgt_lang = "".join(LANGUAGE[LANGUAGE.language == tgt_lang].token.to_list())
+    # dialect = "".join(DIALECT[DIALECT.dialect == dialect].short_name.to_list())
+    src_lang = LANGUAGE[src_lang]
+    tgt_lang = LANGUAGE[tgt_lang]
+    dialect = DIALECT[dialect]
+
+    print(f'Input text: {text} - Time: {datetime.now(tz=TZ)}')
+      
+    text = text.strip()
+      
+    if src_lang == 'krc_Cyrl':
+      text = self._toModel(text)
 
-    
-def toModel(str):
-    str = str.replace("дж", "j") 
-    str = str.replace("Дж", "J") 
-    str = str.replace("ДЖ", "J") 
-    str = str.replace("ж", "j") 
-    str = str.replace("Ж", "J")
-    str = str.replace("себеп", "себеb") 
-    str = str.replace("себеб", "себеb") 
-    str = str.replace("Себеп", "Себеb") 
-    str = str.replace("Себеб", "Себеb") 
-    str = str.replace("тюйюл", "тюл") 
-    str = str.replace("Тюйюл", "Тюл") 
-    str = str.replace("уку", "гылын qуш") 
-    str = str.replace("Уку", "Гылын qуш") 
-    str = str.replace("хораз", "гугурукку") 
-    str = str.replace("Хораз", "Гугурукку") 
-    str = str.replace("юзмез", "qум") 
-    str = str.replace("Юзмез", "Qум") 
-    str = str.replace("арап", "араb") 
-    str = str.replace("араб", "араb") 
-    str = str.replace("Арап", "Араb")
-    str = str.replace("Араб", "Араb")
-    str = str.replace("jиля", "jыла") 
-    str = str.replace("jыла", "jыла") 
-    str = str.replace("jыла", "jыла") 
-    str = str.replace("Jиля", "Jыла") 
-    str = str.replace("Jыла", "Jыла") 
-    str = str.replace("Jыла", "Jыла") 
-    str = str.replace("ярабий", "арабин") 
-    str = str.replace("арабий", "арабин") 
-    str = str.replace("Ярабий", "Арабин") 
-    str = str.replace("Арабий", "Арабин") 
-    str = str.replace("нтта", "нтда") 
-    str = str.replace("ртте", "ртде") 
-    str = str.replace("jамагъат", "jамаgат") 
-    str = str.replace("jамауат", "jамаgат") 
-    str = str.replace("jамагъат", "jамаgат")
-    str = str.replace("jамауат", "jамаgат")
-    str = str.replace("Jамагъат", "Jамаgат") 
-    str = str.replace("Jамауат", "Jамаgат") 
-    str = str.replace("Jамагъат", "Jамаgат") 
-    str = str.replace("Jамаўат", "Jамаgат") 
-    str = str.replace("шуёх", "шох") 
-    str = str.replace("Шуёх", "Шох") 
-    str = str.replace("шёндю", "бусаgат") 
-    str = str.replace("бусагъат", "бусаgат") 
-    str = str.replace("Шёндю", "Бусаgат") 
-    str = str.replace("Бусагъат", "Бусаgат") 
-    str = str.replace("угъай", "оgай")
-    str = str.replace("огъай", "оgай")
-    str = str.replace("Угъай", "Оgай") 
-    str = str.replace("Огъай", "Оgай") 
-    # str = str.replace("терк", "тез") 
-    # str = str.replace("терк", "тез") 
-    str = str.replace("санга", "сенnе") 
-    str = str.replace("сенге", "сенnе") 
-    str = str.replace("сеннге", "сенnе") 
-    str = str.replace("Санга", "Сенnе") 
-    str = str.replace("Сеннге", "Сенnе") 
-    str = str.replace("Сенге", "Сенnе") 
-    str = str.replace("манга", "менnе") 
-    str = str.replace("меннге", "менnе") 
-    str = str.replace("менге", "менnе") 
-    str = str.replace("Манга", "Менnе") 
-    str = str.replace("Меннге", "Менnе") 
-    str = str.replace("Менге", "Менnе") 
-    str = str.replace("аякъ jол", "jахтана") 
-    str = str.replace("аякъ jол", "jахтана") 
-    str = str.replace("jахтана", "jахтана") 
-    str = str.replace("jахтана", "jахтана") 
-    str = str.replace("Аякъ jол", "Jахтана") 
-    str = str.replace("Аякъ jол", "Jахтана") 
-    str = str.replace("Jахтана", "Jахтана") 
-    str = str.replace("Jахтана", "Jахтана") 
-    str = str.replace("къамж", "qамыzh") 
-    str = str.replace("къамыж", "qамыzh") 
-    str = str.replace("Къамж", "Qамыzh") 
-    str = str.replace("Къамыж", "Qамыzh") 
-    str = str.replace("къымыж", "qымыzh") 
-    str = str.replace("къымыж", "qымыzh") 
-    str = str.replace("Къымыж", "Qымыzh") 
-    str = str.replace("Къымыж", "Qымыzh") 
-    str = str.replace("хау", "хо") 
-    str = str.replace("хаў", "хо") 
-    str = str.replace("Хау", "Хо") 
-    str = str.replace("Хаў", "Хо") 
-    str = str.replace("уа", "wa") 
-    str = str.replace("ўа", "wa") 
-    str = str.replace("Уа", "Wa") 
-    str = str.replace("Ўа", "Wa") 
-    str = str.replace("п", "b") 
-    str = str.replace("б", "b") 
-    str = str.replace("къ", "q") 
-    str = str.replace("Къ", "Q") 
-    str = str.replace("КЪ", "Q") 
-    str = str.replace("гъ", "g") 
-    str = str.replace("Гъ", "G") 
-    str = str.replace("ГЪ", "G") 
-    str = str.replace("ц", "ч") 
-    str = str.replace("Ц", "Ч") 
-    str = str.replace("ф", "п") 
-    str = str.replace("сыпат", "сыфат") 
-    str = str.replace("Сыпат", "Сыфат") 
-    str = str.replace("Ф", "П") 
-    str = str.replace("(?<=[аыоуэеиёюя])у(?=[аыоуэеиёюя])|(?<=[аыоуэеиёюя])ў(?=[аыоуэеиёюя])|(?<=[АЫОУЭЕИЁЮЯ])у(?=[АЫОУЭЕИЁЮЯ])|(?<=[АЫОУЭЕИЁЮЯ])ў(?=[АЫОУЭЕИЁЮЯ])", "w")
-    str = str.replace("(?<=[аыоуэеиёюя])у|(?<=[аыоуэеиёюя])ў|(?<=[АЫОУЭЕИЁЮЯ])у|(?<=[АЫОУЭЕИЁЮЯ])ў", "w")
-    # str = str.replace("у(?=[аыоуэеиёюя])|ў(?=[аыоуэеиёюя])|у(?=[АЫОУЭЕИЁЮЯ])|ў(?=[АЫОУЭЕИЁЮЯ])", "w") 
-    # str = str.replace("У(?=[аыоуэеиёюя])|Ў(?=[аыоуэеиёюя])|У(?=[АЫОУЭЕИЁЮЯ])|Ў(?=[АЫОУЭЕИЁЮЯ])", "W") 
-    str = str.replace("zh", "ж") 
-    str = str.replace("нг", "n") 
-    str = str.replace("Нг", "  N")
-    str = str.replace("НГ", "  N")
-    return str
+    # Разбиваем текст на предложения, сохраняя знаки препинания
+    texts_to_translate, original_structure = self._prepareTextAndStructure(text=text)
+    # text бош эсе
+    if len(texts_to_translate) == 0:
+      texts_to_translate = [""]
 
-# 6. Translate function
-def translatePy(text, src_lang='rus_Cyrl', tgt_lang='krc_Cyrl',
-    a=32, b=3, max_input_length=1024, num_beams=3, **kwargs
-):
-    """Turn a text or a list of texts into a list of translations"""
-    tokenizer.src_lang = src_lang
-    tokenizer.tgt_lang = tgt_lang
-    inputs = tokenizer(
-        text, return_tensors='pt', padding=True, truncation=True,
-        max_length=max_input_length
-    )
-    #print(f'Inputs: {inputs}')
-    result = model_translate.generate(
-        **inputs.to(model_translate.device),
-        forced_bos_token_id=tokenizer.convert_tokens_to_ids(tgt_lang),
-        max_new_tokens=int(a + b * inputs.input_ids.shape[1]),
-        num_beams=num_beams, **kwargs
-    )
-    #print(f'Outputs: {result}')
-    return tokenizer.batch_decode(result, skip_special_tokens=True)
-
-
-def translateDisp(text, from_, to, dialect):
-  # print(from_)
-  # print(to)
-  # print(dialect)
-  if dialect == "" or dialect is None:
-    dialect = DIALECT.dialect[0] # "дж\ч"
-  if from_ == "" or from_ is None:
-    from_ = LANGUAGE.language[1] # "Русский язык"
-  if to == "" or to is None:
-    to = LANGUAGE.language[0] # "Къарачай-Малкъар тил"
-    
-  from_ = "".join(LANGUAGE[LANGUAGE.language == from_].token.to_list())
-  to = "".join(LANGUAGE[LANGUAGE.language == to].token.to_list())
-  dialect = "".join(DIALECT[DIALECT.dialect == dialect].short_name.to_list())
+    #print(f'Split text: {texts_to_translate}')
+      
+    translated_texts = self._translate(text=texts_to_translate, src_lang = src_lang, tgt_lang = tgt_lang)
 
-  print(f'Input text: {text} - Time: {datetime.now(tz=TZ)}')
-    
-  text = text.strip()
-    
-  if from_ == 'krc_Cyrl':
-    text = toModel(text)
-
-  # Разбиваем текст на предложения, сохраняя знаки препинания
-  #   .+?[.!?](?:\s|$): Захватывает предложения, которые заканчиваются точкой, восклицательным или вопросительным знаком.
-  # |.+?(?:\n|$): Добавляет поддержку для разрыва строки (\n) или конца текста ($), если предложение не заканчивается знаком препинания.
-  text = re.findall(r'.+?[.!?](?:\s|$)|.+?(?:\n|$)', text)
-  # text бош эсе
-  if len(text) == 0:
-    text = ""
-
-  #print(f'Split text: {text}')
-    
-  str_ = translatePy(text, src_lang = from_, tgt_lang = to)
 
-  #print(f'Translated text: {str_}')
-  str_ = ' '.join(str_).strip()
-  #print(f'Jointed text: {str_}')
+    
+    translated = self._recoverTranslatedToStructure(
+          translated_texts=translated_texts, original_structure=original_structure
+      )
+    #print(f'Translated text: {translated}')
 
-  if to == 'krc_Cyrl':
-    str_ = fromModel(str_, dialect = dialect)
+    if tgt_lang == 'krc_Cyrl':
+      translated = self._fromModel(str=translated, dialect = dialect)
 
-  print(f'Translated text: {str_} - Time: {datetime.now(tz=TZ)}')
-    
-  return str_
+    print(f'Translated text: {translated} - Time: {datetime.now(tz=TZ)}')
+      
+    return translated
   
-# 7. Dictionary function
-def dictionaryDisp(text, from_):
+# Dictionary function
+def dictionaryDisp(text, src_lang):
   
-  if from_ == "" or from_ is None:
-    from_ = LANGUAGE.language[1] # "Русский язык"
+  if src_lang == "" or src_lang is None:
+    src_lang = list(LANGUAGE.keys())[1] # "Русский язык"
       
-  from_ = "".join(LANGUAGE[LANGUAGE.language == from_].token.to_list())
+  src_lang = LANGUAGE[src_lang]
 
   text = text.strip()
   str_l = text.lower()
@@ -454,10 +552,10 @@ def dictionaryDisp(text, from_):
   df_from_to = pd.DataFrame() 
   df_to_from = pd.DataFrame()
   
-  if from_ == 'krc_Cyrl':
+  if src_lang == 'krc_Cyrl':
     df_from_to = dictionary_qm.copy()
     df_to_from = dictionary_ru.copy()
-  elif from_ == 'rus_Cyrl':
+  elif src_lang == 'rus_Cyrl':
     df_from_to = dictionary_ru.copy()
     df_to_from = dictionary_qm.copy()
     
@@ -483,7 +581,7 @@ def dictionaryDisp(text, from_):
   # len(sozluk)
   
   
-# 8. Voice function 
+# Voice function 
 def tts(text):
     file_voice = ''.join(random.choices(string.ascii_letters, k=8))
     file_voice = f'{file_voice}.wav'
@@ -498,7 +596,9 @@ def tts(text):
 
     return file_voice
 
-# 9. Definition ui
+# 5. Definition ui
+translator = Translator(tokenizer=tokenizer, model=model_translate)
+
 _title = "".join(NAMES[NAMES.id == "title"][SYSTEM_LANG].to_list())
 _type = "".join(NAMES[NAMES.id == "type"][SYSTEM_LANG].to_list())
 _from = "".join(NAMES[NAMES.id == "from"][SYSTEM_LANG].to_list())
@@ -523,16 +623,16 @@ with gr.Blocks() as demo:
             # choice_type = gr.Dropdown(
             #   choices = TYPE[SYSTEM_LANG].to_list(), label=_type, value = TYPE[SYSTEM_LANG][0])
             translate_lang_input = gr.Dropdown(
-              choices = LANGUAGE.language.to_list(), label=_from, value = LANGUAGE["language"][1])
+              choices = list(LANGUAGE.keys()), label=_from, value = list(LANGUAGE.keys())[1])
               
         with gr.Column():
           with gr.Row():
             translate_lang_output = gr.Dropdown(
-              choices = LANGUAGE.language.to_list(), label=_to, value = LANGUAGE["language"][0])
+              choices = list(LANGUAGE.keys()), label=_to, value = list(LANGUAGE.keys())[0])
 
             dialect = gr.Dropdown(
               # choices = DIALECT.dialect.to_list(), label=_dialect, value = "дж\ч")
-              choices = DIALECT.dialect.to_list(), label=_dialect, value = DIALECT["dialect"][0])
+              choices = list(DIALECT.keys()), label=_dialect, value = list(DIALECT.keys())[0])
 
       with gr.Row():
         with gr.Column():
@@ -548,7 +648,7 @@ with gr.Blocks() as demo:
         with gr.Column():
           with gr.Row():
             dict_lang_input = gr.Dropdown(
-              choices = LANGUAGE.language.to_list(), label=_from, value = LANGUAGE["language"][1])
+              choices = list(LANGUAGE.keys()), label=_from, value = list(LANGUAGE.keys())[1])
               
 
       with gr.Row():
@@ -570,11 +670,12 @@ with gr.Blocks() as demo:
       tts_button = gr.Button(_sound, variant = 'primary')
 
 
-    translate_button.click(translateDisp, inputs=[translate_text_input, translate_lang_input, translate_lang_output, dialect], outputs=[translate_text_output]) # text, from, to, dialect
+    translate_button.click(translator.translate, inputs=[translate_text_input, translate_lang_input, translate_lang_output, dialect], outputs=[translate_text_output]) # text, from, to, dialect
     dict_button.click(dictionaryDisp, inputs=[dict_text_input, dict_lang_input], outputs=[dict_text_output]) # text, from
     tts_button.click(tts, inputs=[tts_text_input], outputs=[tts_text_output]) # text
 
     gr.Markdown(_annotation)
 
-# 10. Launch
-demo.launch()
+# 6. Launch
+demo.launch()
+# demo.launch(inbrowser=True)