Upload text_split.py

text_split.py

@@ -0,0 +1,134 @@
+import re
+import logging
+from typing import Set, Tuple, List
+
+SENTENCE_SPLITTER = re.compile(r'[^\.?!…]+[\.?!…]*["»“]*')
+
+LAST_WORD_PATTERN = re.compile(r'(?:\b|\d)([a-zа-я]+)\.$', re.IGNORECASE)
+FIRST_WORD_PATTERN = re.compile(r'^\W*(\w+)')
+ENDS_WITH_ONE_LETTER_LAT_AND_DOT_PATTERN = re.compile(r'(\d|\W|\b)([a-zA-Z])\.$')
+HAS_DOT_INSIDE_PATTERN = re.compile(r'[\w]+\.[\w]+\.$', re.IGNORECASE)
+INITIALS_PATTERN = re.compile(r'(\W|\b)([A-ZА-Я]{1})\.$')
+ONLY_RUS_CONSONANTS_PATTERN = re.compile(r'^[бвгджзйклмнпрстфхцчшщ]{1,4}$', re.IGNORECASE)
+STARTS_WITH_EMPTYNESS_PATTERN = re.compile(r'^\s+')
+ENDS_WITH_EMOTION_PATTERN = re.compile(r'[!?…]|\.{2,}\s?[)"«»,“]?$')
+STARTS_WITH_LOWER_PATTERN = re.compile(r'^\s*[–-—-("«]?\s*[a-zа-я]')
+STARTS_WITH_DIGIT_PATTERN = re.compile(r'^\s*\d')
+NUMERATION_PATTERN = re.compile(r'^\W*[IVXMCL\d]+\.$')
+PAIRED_SHORTENING_IN_THE_END_PATTERN = re.compile(r'\b(\w+)\. (\w+)\.\W*$')
+
+JOIN = 0
+MAYBE = 1
+SPLIT = 2
+
+JOINING_SHORTENINGS = {
+    'mr', 'mrs', 'ms', 'dr', 'vs', 'англ', 'итал', 'греч', 'евр', 'араб', 'яп', 'слав', 'кит',
+    'тел', 'св', 'ул', 'устар', 'им', 'г', 'см', 'д', 'стр', 'корп', 'пл', 'пер', 'сокр', 'рис'
+}
+
+SHORTENINGS = {
+    'co', 'corp', 'inc', 'авт', 'адм', 'барр', 'внутр', 'га', 'дифф', 'дол', 'долл', 'зав', 'зам', 'искл',
+    'коп', 'корп', 'куб', 'лат', 'мин', 'о', 'обл', 'обр', 'прим', 'проц', 'р', 'ред', 'руб', 'рус', 'русск',
+    'сан', 'сек', 'тыс', 'эт', 'яз', 'гос', 'мн', 'жен', 'муж', 'накл', 'повел', 'букв', 'шутл', 'ед'
+}
+
+PAIRED_SHORTENINGS = {('и', 'о'), ('т', 'е'), ('т', 'п'), ('у', 'е'), ('н', 'э')}
+
+
+def split_sentences(text: str) -> List[str]:
+    return [x.strip() for x in SENTENCE_SPLITTER.findall(text)]
+
+
+def is_sentence_end(left: str, right: str,
+                    shortenings: Set[str],
+                    joining_shortenings: Set[str],
+                    paired_shortenings: Set[Tuple[str, str]]) -> int:
+    if not STARTS_WITH_EMPTYNESS_PATTERN.match(right):
+        return JOIN
+
+    if HAS_DOT_INSIDE_PATTERN.search(left):
+        return JOIN
+
+    left_last_word = LAST_WORD_PATTERN.search(left)
+    lw = ' '
+    if left_last_word:
+        lw = left_last_word.group(1)
+
+        if lw.lower() in joining_shortenings:
+            return JOIN
+
+        if ONLY_RUS_CONSONANTS_PATTERN.search(lw) and lw[-1].islower():
+            return MAYBE
+
+    pse = PAIRED_SHORTENING_IN_THE_END_PATTERN.search(left)
+    if pse:
+        s1, s2 = pse.groups()
+        if (s1, s2) in paired_shortenings:
+            return MAYBE
+
+    right_first_word = FIRST_WORD_PATTERN.match(right)
+    if right_first_word:
+        rw = right_first_word.group(1)
+        if (lw, rw) in paired_shortenings:
+            return MAYBE
+
+    if ENDS_WITH_EMOTION_PATTERN.search(left) and STARTS_WITH_LOWER_PATTERN.match(right):
+        return JOIN
+
+    initials = INITIALS_PATTERN.search(left)
+    if initials:
+        border, _ = initials.groups()
+        if (border or ' ') not in "°'":
+            return JOIN
+
+    if lw.lower() in shortenings:
+        return MAYBE
+
+    last_letter = ENDS_WITH_ONE_LETTER_LAT_AND_DOT_PATTERN.search(left)
+    if last_letter:
+        border, _ = last_letter.groups()
+        if (border or ' ') not in "°'":
+            return MAYBE
+    if NUMERATION_PATTERN.match(left):
+        return JOIN
+    return SPLIT
+
+
+def split_by_sentences(text: str,
+                      shortenings: Set[str] = SHORTENINGS,
+                      joining_shortenings: Set[str] = JOINING_SHORTENINGS,
+                      paired_shortenings: Set[Tuple[str, str]] = PAIRED_SHORTENINGS) -> List[str]:
+    sentences = []
+    sents = split_sentences(text)
+    si = 0
+    processed_index = 0
+    sent_start = 0
+    while si < len(sents):
+        s = sents[si]
+        span_start = text[processed_index:].index(s) + processed_index
+        span_end = span_start + len(s)
+        processed_index += len(s)
+
+        si += 1
+
+        send = is_sentence_end(text[sent_start: span_end], text[span_end:],
+                               shortenings, joining_shortenings, paired_shortenings)
+        if send == JOIN:
+            continue
+
+        if send == MAYBE:
+            if STARTS_WITH_LOWER_PATTERN.match(text[span_end:]):
+                continue
+            if STARTS_WITH_DIGIT_PATTERN.match(text[span_end:]):
+                continue
+
+        if not text[sent_start: span_end].strip():
+            print(text)
+        sentences.append(text[sent_start: span_end].strip())
+        sent_start = span_end
+        processed_index = span_end
+
+    if sent_start != len(text):
+        if text[sent_start:].strip():
+            sentences.append(text[sent_start:].strip())
+    return sentences