app

app.py

@@ -0,0 +1,52 @@
+import gradio as gr
+
+from bertopic import BERTopic
+from datasets import load_dataset
+from functools import lru_cache
+
+
+def prep_dataset():
+    dataset = load_dataset("OpenAssistant/oasst1", split="train")
+    assistant_ds = dataset.filter(lambda x: x["role"] == "assistant")
+    assistant_ds_en = assistant_ds.filter(lambda x: x["lang"] == "en")
+    return assistant_ds_en["text"]
+
+
+topic_model = BERTopic.load("davanstrien/chat_topics")
+
+fig = topic_model.visualize_topics()
+
+
+def plot_docs():
+    docs = prep_dataset()
+    return topic_model.visualize_documents(docs)
+
+
+def search_topic(text):
+    similar_topics, _ = topic_model.find_topics(text, top_n=5)
+    topic_info = topic_model.get_topic_info()
+    return topic_info[topic_info["Topic"].isin(similar_topics)]
+
+
+def plot_topic_words(num_topics=9, n_words=5):
+    return topic_model.visualize_barchart(top_n_topics=num_topics, n_words=n_words)
+
+
+with gr.Blocks() as demo:
+    with gr.Tab("Topic words"):
+        topic_number = gr.Slider(
+            minimum=3, maximum=20, value=9, step=1, label="Number of topics"
+        )
+        plot = gr.Plot(plot_topic_words())
+        topic_number.change(plot_topic_words, [topic_number], plot)
+    with gr.Tab("Topic search"):
+        text = gr.Textbox(lines=1, label="Search text")
+        df = gr.DataFrame()
+        text.change(search_topic, [text], df)
+    with gr.Tab("Topic distribution"):
+        gr.Plot(fig)
+
+    # with gr.Tab("Doc visualization"):
+    #     gr.Plot(plot_docs())
+
+demo.launch(debug=True)

playground.ipynb

@@ -0,0 +1,32 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.3"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -0,0 +1,340 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --resolver=backtracking
+#
+aiofiles==23.1.0
+    # via gradio
+aiohttp==3.8.4
+    # via
+    #   datasets
+    #   fsspec
+    #   gradio
+aiosignal==1.3.1
+    # via aiohttp
+altair==5.0.1
+    # via gradio
+anyio==3.7.0
+    # via
+    #   httpcore
+    #   starlette
+async-timeout==4.0.2
+    # via aiohttp
+attrs==23.1.0
+    # via
+    #   aiohttp
+    #   jsonschema
+bertopic==0.15.0
+    # via -r requirements.in
+certifi==2023.5.7
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+charset-normalizer==3.1.0
+    # via
+    #   aiohttp
+    #   requests
+click==8.1.3
+    # via
+    #   nltk
+    #   uvicorn
+contourpy==1.0.7
+    # via matplotlib
+cycler==0.11.0
+    # via matplotlib
+cython==0.29.35
+    # via hdbscan
+datasets==2.12.0
+    # via -r requirements.in
+dill==0.3.6
+    # via
+    #   datasets
+    #   multiprocess
+fastapi==0.96.0
+    # via gradio
+fastjsonschema==2.17.1
+    # via nbformat
+ffmpy==0.3.0
+    # via gradio
+filelock==3.12.0
+    # via
+    #   huggingface-hub
+    #   torch
+    #   transformers
+fonttools==4.39.4
+    # via matplotlib
+frozenlist==1.3.3
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2023.5.0
+    # via
+    #   datasets
+    #   gradio-client
+    #   huggingface-hub
+gradio==3.33.1
+    # via -r requirements.in
+gradio-client==0.2.5
+    # via gradio
+h11==0.14.0
+    # via
+    #   httpcore
+    #   uvicorn
+hdbscan==0.8.29
+    # via bertopic
+httpcore==0.17.2
+    # via httpx
+httpx==0.24.1
+    # via
+    #   gradio
+    #   gradio-client
+huggingface-hub==0.15.1
+    # via
+    #   datasets
+    #   gradio
+    #   gradio-client
+    #   sentence-transformers
+    #   transformers
+idna==3.4
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+jinja2==3.1.2
+    # via
+    #   altair
+    #   gradio
+    #   torch
+joblib==1.2.0
+    # via
+    #   hdbscan
+    #   nltk
+    #   pynndescent
+    #   scikit-learn
+jsonschema==4.17.3
+    # via
+    #   altair
+    #   nbformat
+jupyter-core==5.3.0
+    # via nbformat
+kiwisolver==1.4.4
+    # via matplotlib
+linkify-it-py==2.0.2
+    # via markdown-it-py
+llvmlite==0.40.0
+    # via
+    #   numba
+    #   pynndescent
+markdown-it-py[linkify]==2.2.0
+    # via
+    #   gradio
+    #   mdit-py-plugins
+markupsafe==2.1.3
+    # via
+    #   gradio
+    #   jinja2
+matplotlib==3.7.1
+    # via gradio
+mdit-py-plugins==0.3.3
+    # via gradio
+mdurl==0.1.2
+    # via markdown-it-py
+mpmath==1.3.0
+    # via sympy
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+multiprocess==0.70.14
+    # via datasets
+nbformat==5.9.0
+    # via -r requirements.in
+networkx==3.1
+    # via torch
+nltk==3.8.1
+    # via sentence-transformers
+numba==0.57.0
+    # via
+    #   pynndescent
+    #   umap-learn
+numpy==1.24.3
+    # via
+    #   altair
+    #   bertopic
+    #   contourpy
+    #   datasets
+    #   gradio
+    #   hdbscan
+    #   matplotlib
+    #   numba
+    #   pandas
+    #   pyarrow
+    #   scikit-learn
+    #   scipy
+    #   sentence-transformers
+    #   torchvision
+    #   transformers
+    #   umap-learn
+orjson==3.9.0
+    # via gradio
+packaging==23.1
+    # via
+    #   datasets
+    #   gradio-client
+    #   huggingface-hub
+    #   matplotlib
+    #   plotly
+    #   transformers
+pandas==2.0.2
+    # via
+    #   altair
+    #   bertopic
+    #   datasets
+    #   gradio
+pillow==9.5.0
+    # via
+    #   gradio
+    #   matplotlib
+    #   torchvision
+platformdirs==3.5.1
+    # via jupyter-core
+plotly==5.14.1
+    # via bertopic
+pyarrow==12.0.0
+    # via datasets
+pydantic==1.10.8
+    # via
+    #   fastapi
+    #   gradio
+pydub==0.25.1
+    # via gradio
+pygments==2.15.1
+    # via gradio
+pynndescent==0.5.10
+    # via umap-learn
+pyparsing==3.0.9
+    # via matplotlib
+pyrsistent==0.19.3
+    # via jsonschema
+python-dateutil==2.8.2
+    # via
+    #   matplotlib
+    #   pandas
+python-multipart==0.0.6
+    # via gradio
+pytz==2023.3
+    # via pandas
+pyyaml==6.0
+    # via
+    #   datasets
+    #   gradio
+    #   huggingface-hub
+    #   transformers
+regex==2023.6.3
+    # via
+    #   nltk
+    #   transformers
+requests==2.31.0
+    # via
+    #   datasets
+    #   fsspec
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   responses
+    #   torchvision
+    #   transformers
+responses==0.18.0
+    # via datasets
+safetensors==0.3.1
+    # via -r requirements.in
+scikit-learn==1.2.2
+    # via
+    #   bertopic
+    #   hdbscan
+    #   pynndescent
+    #   sentence-transformers
+    #   umap-learn
+scipy==1.10.1
+    # via
+    #   hdbscan
+    #   pynndescent
+    #   scikit-learn
+    #   sentence-transformers
+    #   umap-learn
+semantic-version==2.10.0
+    # via gradio
+sentence-transformers==2.2.2
+    # via bertopic
+sentencepiece==0.1.99
+    # via sentence-transformers
+six==1.16.0
+    # via python-dateutil
+sniffio==1.3.0
+    # via
+    #   anyio
+    #   httpcore
+    #   httpx
+starlette==0.27.0
+    # via fastapi
+sympy==1.12
+    # via torch
+tenacity==8.2.2
+    # via plotly
+threadpoolctl==3.1.0
+    # via scikit-learn
+tokenizers==0.13.3
+    # via transformers
+toolz==0.12.0
+    # via altair
+torch==2.0.1
+    # via
+    #   sentence-transformers
+    #   torchvision
+torchvision==0.15.2
+    # via sentence-transformers
+tqdm==4.65.0
+    # via
+    #   bertopic
+    #   datasets
+    #   huggingface-hub
+    #   nltk
+    #   sentence-transformers
+    #   transformers
+    #   umap-learn
+traitlets==5.9.0
+    # via
+    #   jupyter-core
+    #   nbformat
+transformers==4.29.2
+    # via sentence-transformers
+typing-extensions==4.6.3
+    # via
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   pydantic
+    #   torch
+tzdata==2023.3
+    # via pandas
+uc-micro-py==1.0.2
+    # via linkify-it-py
+umap-learn==0.5.3
+    # via bertopic
+urllib3==2.0.2
+    # via
+    #   requests
+    #   responses
+uvicorn==0.22.0
+    # via gradio
+websockets==11.0.3
+    # via
+    #   gradio
+    #   gradio-client
+xxhash==3.2.0
+    # via datasets
+yarl==1.9.2
+    # via aiohttp