Merge remote-tracking branch 'origin/saied' into develop

src/data_utils.py

@@ -22,6 +22,14 @@ def filter_by_num_tokens(text, gt=64):
 def filter_by_num_sents(text, gt=2):
     return len(sent_tokenize(text)) > gt
 
+def remove_adds(text,ratio=50):
+    comma = text.split(",")
+    colon = re.findall(r'(?:([^\W]+):([^\W]+))',text)
+    virgool = text.split("،")
+    length_add = len(comma)+len(colon)+len(virgool)
+
+    return True if length_add < ratio else False
+
 
 def normalizer(text, do_lowercase=False):
     text = normalize(text)

src/normalizer.py

@@ -25,6 +25,13 @@ def multiple_replace(text, chars_to_mapping):
     pattern = "|".join(map(re.escape, chars_to_mapping.keys()))
     return re.sub(pattern, lambda m: chars_to_mapping[m.group()], str(text))
 
+def remove_tags(text):
+    tag = "برچسب ها :"
+    try:
+        text = text[:text.index(tag)]
+        return text
+    except:
+        return text
 
 def clean_url(text):
     # removing html tags
@@ -79,6 +86,7 @@ def normalize(text, zwnj="\u200c", tokenized=False):
     text = DOUBLE_QUOTE_REGEX.sub('"', text)
     text = CURRENCY_REGEX.sub(r" \1 ", text)
     text = clean_url(text)
+    text = remove_tags(text)
     text = URL_REGEX.sub(" ", text)
     text = EMAIL_REGEX.sub(" ", text)
     text = PHONE_REGEX.sub(r" \1 ", text)