Hello gpt2-persian

README.md

@@ -0,0 +1,33 @@
+# GPT2 - Persian
+
+
+## Scripts
+
+### Normalizer
+
+```python
+from src.normalizer import normalize
+
+input_text = "ὑ蕉Ұ제ṅ尘̲改座◦花芝秀黄天자埃澤ಿ ˈazbab اینجا ایران خانه‌شما است؟!۱۲۳۱۲۳۱۳۱۲ اَلْحُرُوفُ ٱلْعَرَبِیَّة"input_text = normalize(input_text)
+print(normalize(input_text))
+```
+
+Output:
+```text
+azbab اینجا ایران خانه‌شما است ؟ ! 1231231312 الحروف لعربیه
+```
+
+### Training tokenizer
+
+```bash
+python train_tokenizer.py --dataset_name oscar --dataset_config_name unshuffled_deduplicated_als --vocab_size 42000
+```
+
+### Configuration
+
+```bash
+python create_config.py --name_or_path gpt2-medium --params '{"vocab_size": 42000}'
+```
+
+
+

src/create_config.py

@@ -0,0 +1,74 @@
+import ast
+import logging
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+
+from transformers import (
+    HfArgumentParser,
+    AutoConfig
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ConfigArguments:
+    """
+    Arguments to which config we are going to set up.
+    """
+    output_dir: str = field(
+        default=".",
+        metadata={"help": "The output directory where the config will be written."},
+    )
+    name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The model checkpoint for weights initialization."
+                    "Don't set if you want to train a model from scratch."
+        },
+    )
+    params: Optional[str] = field(
+        default=None,
+        metadata={"help": "Custom configuration for the specific `name_or_path`"}
+    )
+
+    def __post_init__(self):
+        if self.params:
+            try:
+                self.params = ast.literal_eval(self.params)
+            except Exception as e:
+                print(f"Your custom parameters do not acceptable due to {e}")
+
+
+def main():
+    parser = HfArgumentParser([ConfigArguments])
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        config_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))[0]
+    else:
+        config_args = parser.parse_args_into_dataclasses()[0]
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+    logger.setLevel(logging.INFO)
+
+    logger.info(f"Setting up configuration {config_args.name_or_path} with extra params {config_args.params}")
+
+    if config_args.params and isinstance(config_args.params, dict):
+        config = AutoConfig.from_pretrained(config_args.name_or_path, **config_args.params)
+    else:
+        config = AutoConfig.from_pretrained(config_args.name_or_path)
+
+    logger.info(f"Your configuration saved here {config_args.output_dir}/config.json")
+    config.save_pretrained(config_args.output_dir)
+
+
+if __name__ == '__main__':
+    main()

src/dictionary.py

@@ -0,0 +1,133 @@
+characters = {
+    "ك": "ک",
+    "دِ": "د",
+    "بِ": "ب",
+    "زِ": "ز",
+    "ذِ": "ذ",
+    "شِ": "ش",
+    "سِ": "س",
+    "ى": "ی",
+    "ي": "ی",
+    "ؤ": "و",
+    "ے": "ی",
+    "ۀ": "ه",
+    "ﭘ": "پ",
+    "ﮐ": "ک",
+    "ﯽ": "ی",
+    "ﺎ": "ا",
+    "ﺑ": "ب",
+    "ﺘ": "ت",
+    "ﺧ": "خ",
+    "ﺩ": "د",
+    "ﺱ": "س",
+    "ﻀ": "ض",
+    "ﻌ": "ع",
+    "ﻟ": "ل",
+    "ﻡ": "م",
+    "ﻢ": "م",
+    "ﻪ": "ه",
+    "ﻮ": "و",
+    "ﺍ": "ا",
+    "ة": "ه",
+    "ﯾ": "ی",
+    "ﯿ": "ی",
+    "ﺒ": "ب",
+    "ﺖ": "ت",
+    "ﺪ": "د",
+    "ﺮ": "ر",
+    "ﺴ": "س",
+    "ﺷ": "ش",
+    "ﺸ": "ش",
+    "ﻋ": "ع",
+    "ﻤ": "م",
+    "ﻥ": "ن",
+    "ﻧ": "ن",
+    "ﻭ": "و",
+    "ﺭ": "ر",
+    "ﮔ": "گ",
+    "إ": "ا",
+    "ٕ": " ",
+    "ھ": "ه",
+    "...": ".",
+    "…": ".",
+    "-": " - ",
+    "هٔ": "ه",
+    "ﻯ": "ی",
+    "ﻛ": "ک",
+    "ﭼ": "چ",
+    "ﺓ": "ه",
+    "ﻴ": "ی",
+    "ﻊ": "ع",
+    "ﮬ": "ه",
+    "ﺟ": "ج",
+    "ﺳ": "س",
+    "ﻦ": "ن",
+    "ﺬ": "ذ",
+    "ﺋ": "ئ",
+    "ﷲ": "لله",
+    "ﺞ": "ج",
+    "ﺙ": "ث",
+    "ﻗ": "ق",
+    "ﮪ": "ه",
+    "ﺰ": "ز",
+    "ﯼ": "ی",
+    "ٺ": "ت",
+    "ﺻ": "ص",
+    "ﻂ": "ط",
+    "ﻣ": "م",
+    "ﻈ": "ظ",
+    "ﺐ": "ب",
+    "ﻍ": "غ",
+    "ݸ": "و",
+    "ﻨ": "ن",
+    "ﻝ": "ل",
+    "ﻩ": "ه",
+    "ﻲ": "ی",
+    "ﻐ": "غ",
+    "ﺲ": "س",
+    "ﺁ": "آ",
+    "ڔ": "ر",
+    "ﺫ": "ذ",
+    "ﭻ": "چ",
+    "ﺠ": "ج",
+    "ﯙ": "و",
+    "ﮏ": "ک",
+    "ﺣ": "ح",
+    "ﺝ": "ج",
+    "ﺼ": "ص",
+    "ﻳ": "ی",
+    "ﻘ": "ق",
+    "ﺨ": "خ",
+    "ﻔ": "ف",
+    "ﻎ": "غ",
+    "ئ": "ی",
+    "ﻓ": "ف",
+    "ﻕ": "ق",
+    "ﮋ": "ژ",
+    "ﺗ": "ت",
+    "ﻁ": "ط",
+    "ﺯ": "ز",
+    "ﮕ": "گ",
+    "ﺌ": "ئ",
+    "ﺵ": "ش",
+    "ۮ": "د",
+    "ﻫ": "ه",
+    "ﻬ": "ه",
+    "ﻏ": "غ",
+    "ﻰ": "ی",
+    "﷼": "ریال",
+    "ﺿ": "ض",
+    "ﺛ": "ث",
+    "ݐ": "پ",
+    "ﺏ": "ب",
+    "ﭙ": "پ",
+    "ﭽ": "چ",
+    "ﺜ": "ث",
+    "ﻃ": "ط",
+    "ۂ": "ه",
+    "ﻑ": "ف",
+    "ﺕ": "ت",
+    "ﻞ": "ل",
+}
+
+special_tokens = {}

src/normalizer.py

@@ -0,0 +1,93 @@
+import hazm
+import re
+
+from regexes.currency import CURRENCY_REGEX
+from regexes.email import EMAIL_REGEX
+from regexes.latin import LATIN_REGEX
+from regexes.latin import LATIN_REGEX, LATIN_WITH_SPECIAL_REGEX
+from regexes.number import NUMBERS_REGEX
+from regexes.phone import PHONE_REGEX
+from regexes.quote import DOUBLE_QUOTE_REGEX, SINGLE_QUOTE_REGEX
+from regexes.url import URL_REGEX
+from regexes.persian import PERSIAN_REGEX
+from regexes.punk import PUNK_REGEX
+import dictionary
+
+
+def make_trans(list_a, list_b):
+    return dict((ord(a), b) for a, b in zip(list_a, list_b))
+
+
+def multiple_replace(text, chars_to_mapping):
+    pattern = "|".join(map(re.escape, chars_to_mapping.keys()))
+    return re.sub(pattern, lambda m: chars_to_mapping[m.group()], str(text))
+
+
+ar2fa_digits = make_trans("٠١٢٣٤٥٦٧٨٩٪", "۰۱۲۳۴۵۶۷۸۹٪")
+fa2en_digits = make_trans("۰۱۲۳۴۵۶۷۸۹٪", "0123456789%")
+normalizer = hazm.Normalizer(persian_numbers=True)
+
+
+def normalize(text, zwnj="\u200c", tokenized=False):
+    text = text.replace("\n", " ").replace("\t", " ")
+    text = re.sub(r"\u200c+", "\u200c", text)
+
+    text = normalizer.normalize(text)
+
+    if len(dictionary.characters) > 0:
+        text = multiple_replace(text, dictionary.characters)
+
+    text = text.translate(ar2fa_digits)
+    text = text.translate(fa2en_digits)
+
+    text = SINGLE_QUOTE_REGEX.sub("'", text)
+    text = DOUBLE_QUOTE_REGEX.sub('"', text)
+    text = CURRENCY_REGEX.sub(r" \1 ", text)
+    text = URL_REGEX.sub(r" \1 ", text)
+    text = EMAIL_REGEX.sub(r" \1 ", text)
+    text = PHONE_REGEX.sub(r" \1 ", text)
+    text = NUMBERS_REGEX.sub(r" \1 ", text)
+    text = LATIN_REGEX.sub(r" \1 ", text)
+    text = PUNK_REGEX.sub(r" \1 ", text)
+
+    # Allow only english and persian characters
+    text = re.sub(PERSIAN_REGEX, " ", text)
+
+    text = text.replace(f" {zwnj} ", f"{zwnj}")
+    text = text.replace(f"{zwnj} ", f"{zwnj}")
+    text = text.replace(f" {zwnj}", f"{zwnj}")
+
+    if len(dictionary.special_tokens) > 0:
+        text = multiple_replace(text, dictionary.special_tokens)
+
+    tokens = []
+    for token in text.split():
+        token = token.strip()
+        if token:
+            if token.startswith(zwnj) and token.endswith(zwnj):
+                token = token[1:-1]
+            if token.startswith(zwnj):
+                token = token[1:]
+            elif token.endswith(zwnj):
+                token = token[:-1]
+            else:
+                token = token
+
+            tokens.append(token)
+
+    if tokenized:
+        return tokens
+
+    return " ".join(tokens)
+
+
+if __name__ == '__main__':
+    import textwrap
+
+    input_text = "دارهٔ تحقیقات فدرال در سال ۱۹۰۸ به نام ادارهٔ تحقیقات (BOI یا BI) بنیان‌گذاری شد. نام این سازمان در سال ۱۹۳۵ به ادارهٔ تحقیقات فدرال تغییر یافت. دفتر مرکزی اف‌بی‌آی در ساختمان جی. ادگار هوور در شهر واشینگتن، دی.سی. واقع شده‌است."
+    input_text = "یونان (به یونانی: Ελλάδα, اِلادا)"
+    input_text = "نسخهٔ"
+    input_text = "ὑ蕉Ұ제ṅ尘̲改座◦花芝秀黄天자埃澤ಿ ˈazbab اینجا ایران خانه‌شما است؟!۱۲۳۱۲۳۱۳۱۲ اَلْحُرُوفُ ٱلْعَرَبِیَّة"
+    input_text = normalize(input_text)
+    print(textwrap.fill(input_text))
+    print(normalize(input_text, tokenized=True))

src/regexes/currency.py

@@ -0,0 +1,23 @@
+import re
+
+CURRENCIES = {
+    "$": "USD",
+    "zł": "PLN",
+    "£": "GBP",
+    "¥": "JPY",
+    "฿": "THB",
+    "₡": "CRC",
+    "₦": "NGN",
+    "₩": "KRW",
+    "₪": "ILS",
+    "₫": "VND",
+    "€": "EUR",
+    "₱": "PHP",
+    "₲": "PYG",
+    "₴": "UAH",
+    "₹": "INR",
+    "﷼": "IRR",
+}
+CURRENCY_REGEX = re.compile(
+    "({})+".format("|".join(re.escape(c) for c in CURRENCIES.keys()))
+)

src/regexes/email.py

@@ -0,0 +1,6 @@
+import re
+
+EMAIL_REGEX = re.compile(
+    r"(?:^|(?<=[^\w@.)]))([\w+-](\.(?!\.))?)*?[\w+-](@|[(<{\[]at[)>}\]])(?:(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)(?:\.(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)*(?:\.(?:[a-z\\u00a1-\\uffff]{2,}))",
+    flags=re.IGNORECASE | re.UNICODE,
+)

src/regexes/latin.py

@@ -0,0 +1,13 @@
+import re
+
+LATIN_WITH_SPECIAL_REGEX = re.compile(
+    r"(\b(?!URL|EMAIL|PHONE|NUMBER|CUR|LATIN\b)[0-9a-zA-Z]+)"
+)
+
+LATIN_REGEX = re.compile(
+    r"([0-9a-zA-Z]+)"
+)
+
+LATIN_SPACES_REGEX = re.compile(
+    r"([0-9a-zA-Z])"
+)

src/regexes/number.py

@@ -0,0 +1,5 @@
+import re
+
+NUMBERS_REGEX = re.compile(
+    r"(?:^|(?<=[^\w,.]))[+–-]?(([1-9]\d{0,2}(,\d{3})+(\.\d*)?)|([1-9]\d{0,2}([ .]\d{3})+(,\d*)?)|(\d*?[.,]\d+)|\d+)(?:$|(?=\b))"
+)

src/regexes/persian.py

@@ -0,0 +1,19 @@
+import re
+
+
+PERSIAN_ALPHA = "ءآئابتثجحخدذرزسشصضطظعغفقلمنهوپچژکگیە"  # noqa: E501
+PERSIAN_DIGIT = "۰۱۲۳۴۵۶۷۸۹"
+
+
+ZWNJ = "\u200c"
+PUNK = '\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\]\^\_\`\{\|\}\~\«\»\؟\:\×\٬\٫\﷼\٪\،'
+
+PERSIAN = (
+    "a-zA-Z0-9" +
+    PERSIAN_ALPHA +
+    PERSIAN_DIGIT +
+    ZWNJ +
+    PUNK
+)
+
+PERSIAN_REGEX = r"[^" + PERSIAN + "+]"

src/regexes/phone.py

@@ -0,0 +1,6 @@
+import re
+
+
+PHONE_REGEX = re.compile(
+    r"((?:^|(?<=[^\w)]))(((\+?[01])|(\+\d{2}))[ .-]?)?(\(?\d{3,4}\)?/?[ .-]?)?(\d{3}[ .-]?\d{4})(\s?(?:ext\.?|[#x-])\s?\d{2,6})?(?:$|(?=\W)))|\+?\d{4,5}[ .-/]\d{6,9}"
+)

src/regexes/punk.py

@@ -0,0 +1,5 @@
+import re
+
+PUNK_REGEX = re.compile(
+    r"([\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\=\?\@\[\\\]\^\_\`\{\|\}\~\«\»\⸮\؟\،\٬\٫\؛])"
+)

src/regexes/quote.py

@@ -0,0 +1,25 @@
+import re
+
+
+strange_double_quotes = [
+    "«",
+    "‹",
+    "»",
+    "›",
+    "„",
+    "“",
+    "‟",
+    "”",
+    "❝",
+    "❞",
+    "❮",
+    "❯",
+    "〝",
+    "〞",
+    "〟",
+    "＂",
+]
+strange_single_quotes = ["‘", "‛", "’", "❛", "❜", "`", "´", "‘", "’"]
+
+DOUBLE_QUOTE_REGEX = re.compile("|".join(strange_double_quotes))
+SINGLE_QUOTE_REGEX = re.compile("|".join(strange_single_quotes))

src/regexes/url.py

@@ -0,0 +1,38 @@
+import re
+
+URL_REGEX = re.compile(
+    r"(?:^|(?<![\w\/\.]))"
+    # protocol identifier
+    # r"(?:(?:https?|ftp)://)"  <-- alt?
+    r"(?:(?:https?:\/\/|ftp:\/\/|www\d{0,3}\.))"
+    # user:pass authentication
+    r"(?:\S+(?::\S*)?@)?" r"(?:"
+    # IP address exclusion
+    # private & local networks
+    r"(?!(?:10|127)(?:\.\d{1,3}){3})"
+    r"(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})"
+    r"(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})"
+    # IP address dotted notation octets
+    # excludes loopback network 0.0.0.0
+    # excludes reserved space >= 224.0.0.0
+    # excludes network & broadcast addresses
+    # (first & last IP address of each class)
+    r"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])"
+    r"(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}"
+    r"(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))"
+    r"|"
+    # host name
+    r"(?:(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)"
+    # domain name
+    r"(?:\.(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)*"
+    # TLD identifier
+    r"(?:\.(?:[a-z\\u00a1-\\uffff]{2,}))" r"|" r"(?:(localhost))" r")"
+    # port number
+    r"(?::\d{2,5})?"
+    # resource path
+    r"(?:\/[^\)\]\}\s]*)?",
+    # r"(?:$|(?![\w?!+&\/\)]))",
+    # @jfilter: I removed the line above from the regex because I don't understand what it is used for, maybe it was useful?
+    # But I made sure that it does not include ), ] and } in the URL.
+    flags=re.UNICODE | re.IGNORECASE,
+)

src/requirements.txt

@@ -0,0 +1,6 @@
+datasets >= 1.1.3
+jax>=0.2.8
+jaxlib>=0.1.59
+flax>=0.3.4
+optax>=0.0.8
+hazm

src/train_tokenizer.py

@@ -0,0 +1,132 @@
+import ast
+import logging
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+
+from datasets import load_dataset
+from tokenizers import ByteLevelBPETokenizer
+from transformers import (
+    HfArgumentParser,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TokenizerArguments:
+    """
+    Arguments to which tokenizer we are going to set up.
+    """
+
+    output_dir: str = field(
+        default=".",
+        metadata={"help": "The output directory where the config will be written."},
+    )
+    dataset_name: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_config_name: Optional[str] = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    special_tokens: Optional[List[str]] = field(
+        default=None,
+        metadata={"help": "The list of special tokens that you want to add in your training."}
+    )
+    vocab_size: Optional[int] = field(
+        default=50257,
+        metadata={"help": "The size of the final vocabulary, including all tokens and alphabet"}
+    )
+    min_frequency: Optional[int] = field(
+        default=2,
+        metadata={"help": "The minimum frequency a pair should have in order to be merged"}
+    )
+    show_progress: Optional[bool] = field(
+        default=True,
+        metadata={"help": "Whether to show progress bars while training"}
+    )
+
+    def __post_init__(self):
+        if self.special_tokens is None:
+            self.special_tokens = [
+                "<s>", "<pad>", "</s>", "<unk>", "<mask>",
+                "<|endoftext|>", "<|startoftext|>",
+                "<sep>", "<cls>", "<nl>", "<tab>", "<zwnj>"
+            ]
+
+        self.special_tokens = self.special_tokens + [f"[U{i}]" for i in range(1, 21)]
+        if self.dataset_name is None and self.train_file is None:
+            raise ValueError("Need either a dataset name or a training file.")
+        else:
+            if self.train_file is not None:
+                extension = self.train_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
+
+
+def main():
+    parser = HfArgumentParser([TokenizerArguments])
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        tokenizer_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))[0]
+    else:
+        tokenizer_args = parser.parse_args_into_dataclasses()[0]
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+    logger.setLevel(logging.INFO)
+
+    logger.info(f"Training tokenizer")
+
+    if tokenizer_args.dataset_name is not None:
+        dataset = load_dataset(
+            tokenizer_args.dataset_name,
+            tokenizer_args.dataset_config_name,
+            cache_dir=tokenizer_args.cache_dir,
+            split="train"
+        )
+    else:
+        data_files = {"train": tokenizer_args.train_file}
+        extension = tokenizer_args.train_file.split(".")[-1]
+        if extension == "txt":
+            extension = "text"
+
+        dataset = load_dataset(
+            extension,
+            data_files=data_files,
+            delimiter="\t",
+            cache_dir=tokenizer_args.cache_dir,
+        )
+
+    tokenizer = ByteLevelBPETokenizer()
+
+    def batch_iterative(batch_size=1000):
+        for i in range(0, len(dataset), batch_size):
+            yield dataset[i: i + batch_size]["text"]
+
+    tokenizer.train_from_iterator(
+        batch_iterative(),
+        vocab_size=tokenizer_args.vocab_size,
+        special_tokens=tokenizer_args.special_tokens,
+        min_frequency=tokenizer_args.min_frequency,
+        show_progress=tokenizer_args.show_progress,
+    )
+
+    logger.info(f"Your tokenizer saved here {tokenizer_args.output_dir}/tokenizer")
+    os.makedirs(tokenizer_args.output_dir, exist_ok=True)
+    tokenizer.save_model(tokenizer_args.output_dir)
+    tokenizer.save(f"{tokenizer_args.output_dir}/tokenizer.json", pretty=True)
+
+
+if __name__ == '__main__':
+    main()