init

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+.idea/
+.venv/
+.ruff_cache/
+
+flagged/

Dockerfile

@@ -0,0 +1,61 @@
+FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    git \
+    git-lfs \
+    wget \
+    curl \
+    # python build dependencies \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libncursesw5-dev \
+    xz-utils \
+    tk-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
+    # gradio dependencies \
+    ffmpeg \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:${PATH}
+WORKDIR ${HOME}/app
+
+RUN curl https://pyenv.run | bash
+ENV PATH=${HOME}/.pyenv/shims:${HOME}/.pyenv/bin:${PATH}
+ARG PYTHON_VERSION=3.10.12
+RUN pyenv install ${PYTHON_VERSION} && \
+    pyenv global ${PYTHON_VERSION} && \
+    pyenv rehash && \
+    pip install --no-cache-dir -U pip setuptools wheel && \
+    pip install packaging ninja
+
+COPY --chown=1000 ./requirements.txt /tmp/requirements.txt
+RUN pip install youtokentome --no-build-isolation
+RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt
+
+
+COPY --chown=1000 . ${HOME}/app
+ENV PYTHONPATH=${HOME}/app \
+    PYTHONUNBUFFERED=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_THEME=huggingface \
+    SYSTEM=spaces
+
+CMD ["python", "app.py"]

README.md

@@ -1,11 +1,33 @@
 ---
-title: Punctuation Uk
-emoji: 😻
+title: Restore Punctuation and Capitalization for Ukrainian
+emoji: ✍️
 colorFrom: blue
-colorTo: pink
-sdk: docker
+colorTo: yellow
+sdk: gradio
+app_file: app.py
 pinned: false
-license: apache-2.0
+sdk_version: 4.39.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Install
+
+```shell
+uv venv --python 3.10
+
+source .venv/bin/activate
+
+uv pip install Cython packaging
+
+uv pip install youtokentome --no-build-isolation
+
+SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL=True uv pip install -r requirements.txt
+
+# in development mode
+uv pip install -r requirements-dev.txt
+```
+
+## Run
+
+```shell
+python app.py
+```

app.py

@@ -0,0 +1,177 @@
+import sys
+import time
+
+# import torch
+import gradio as gr
+
+from nemo import __version__ as nemo_version
+from nemo.collections.nlp.models import PunctuationCapitalizationModel
+
+# Config
+model_name = "ai-forever/T5-large-spell"
+
+concurrency_limit = 5
+
+# Torch
+# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+# torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+
+# Load the model
+model = PunctuationCapitalizationModel.from_pretrained(
+    "dchaplinsky/punctuation_uk_bert"
+)
+
+examples = [
+    "тема про яку не люблять говорити офіційні джерела у генштабі і міноборони це хімічна зброя окупанти вже тривалий час використовують хімічну зброю заборонену",
+    "всіма конвенціями якщо спочатку це були гранати з дронів то тепер фіксують випадки застосування",
+    "хімічних снарядів причому склад отруйної речовони різний а отже й наслідки для наших військових теж різні",
+    "використовує на фронті все що має і хімічна зброя не вийняток тож з чим маємо справу розбиралася марія моганисян",
+    "двох тисяч випадків застосування росіянами боєприпасів споряджених небезпечними хімічними речовинами",
+    "на всі писані норми марія моганисян олександр моторний спецкор марафон єдині новини",
+]
+
+title = "Restore Punctuation and Capitalization for Ukrainian"
+
+# https://www.tablesgenerator.com/markdown_tables
+authors_table = """
+## Authors
+
+Follow them on social networks and **contact** if you need any help or have any questions:
+
+| <img src="https://avatars.githubusercontent.com/u/7875085?v=4" width="100"> **Yehor Smoliakov** |
+|-------------------------------------------------------------------------------------------------|
+| https://t.me/smlkw in Telegram                                                                  |
+| https://x.com/yehor_smoliakov at X                                                              |
+| https://github.com/egorsmkv at GitHub                                                           |
+| https://huggingface.co/Yehor at Hugging Face                                                    |
+| or use egorsmkv@gmail.com                                                                       |
+""".strip()
+
+description_head = f"""
+# {title}
+
+## Overview
+
+This space uses https://huggingface.co/dchaplinsky/punctuation_uk_bert model.
+
+Paste the text you want to enhance.
+""".strip()
+
+description_foot = f"""
+{authors_table}
+""".strip()
+
+enhanced_text_value = """
+Enhanced text will appear here.
+
+Choose **an example** below the Enhance button or paste **your text**.
+""".strip()
+
+tech_env = f"""
+#### Environment
+
+- Python: {sys.version}
+""".strip()
+
+tech_libraries = f"""
+#### Libraries
+
+- nemo: {nemo_version}
+- gradio: {gr.__version__}
+""".strip()
+
+
+def inference(text, progress=gr.Progress()):
+    if not text:
+        raise gr.Error("Please paste your text.")
+
+    gr.Info("Starting enhancing", duration=2)
+
+    progress(0, desc="Enhancing...")
+
+    results = []
+
+    sentences = [
+        text,
+    ]
+
+    for sentence in progress.tqdm(sentences, desc="Enhancing...", unit="sentence"):
+        sentence = sentence.strip()
+
+        if len(sentence) == 0:
+            continue
+
+        t0 = time.time()
+
+        predictions = model.add_punctuation_capitalization([sentence])
+
+        if not predictions:
+            predictions = "-"
+
+        elapsed_time = round(time.time() - t0, 2)
+
+        enhanced_text = "\n".join(predictions)
+
+        if sentence != enhanced_text:
+            enhanced_text = enhanced_text.strip()
+            results.append(
+                {
+                    "sentence": sentence,
+                    "enhanced_text": enhanced_text,
+                    "elapsed_time": elapsed_time,
+                }
+            )
+
+    gr.Info("Finished!", duration=2)
+
+    result_texts = []
+
+    for result in results:
+        result_texts.append(f'> {result["enhanced_text"]}')
+        result_texts.append("\n")
+
+    sum_elapsed_text = sum([result["elapsed_time"] for result in results])
+    result_texts.append(f"Elapsed time: {sum_elapsed_text} seconds")
+
+    return "\n".join(result_texts)
+
+
+demo = gr.Blocks(
+    title=title,
+    analytics_enabled=False,
+    theme=gr.themes.Base(),
+)
+
+with demo:
+    gr.Markdown(description_head)
+
+    gr.Markdown("## Usage")
+
+    with gr.Row():
+        text = gr.Textbox(label="Text", autofocus=True, max_lines=1)
+        enhanced_text = gr.Textbox(
+            label="Enhanced text",
+            placeholder=enhanced_text_value,
+            show_copy_button=True,
+        )
+
+    gr.Button("Enhance").click(
+        inference,
+        concurrency_limit=concurrency_limit,
+        inputs=text,
+        outputs=enhanced_text,
+    )
+
+    with gr.Row():
+        gr.Examples(label="Choose an example", inputs=text, examples=examples)
+
+    gr.Markdown(description_foot)
+
+    gr.Markdown("### Gradio app uses the following technologies:")
+    with gr.Row():
+        gr.Markdown(tech_env)
+        gr.Markdown(tech_libraries)
+
+if __name__ == "__main__":
+    demo.queue()
+    demo.launch()

requirements-dev.txt

@@ -0,0 +1 @@
+ruff

requirements.txt

@@ -0,0 +1,9 @@
+gradio
+
+Cython
+packaging
+
+huggingface_hub<0.22.1
+numpy<2
+
+nemo_toolkit[nlp]