logic.py

from tabula import read_pdf
from bs4 import BeautifulSoup
import requests

from llama_cpp import Llama
from bertopic.representation import KeyBERTInspired, LlamaCPP
from sentence_transformers import SentenceTransformer
from umap import UMAP
from hdbscan import HDBSCAN
from bertopic import BERTopic

import PIL
import numpy as np
import datamapplot
import re

def get_links():
    #reads table from pdf file
    dfs = read_pdf("Artificial_Intelligence_Bookmarks_AwesomeList.pdf",pages="all") #upload pdf file
    links = dfs[0]['Unnamed: 2'].to_list()
    for i in range(len(dfs)-1):
      links.extend(dfs[i+1]['Url'].to_list())
    return links

#--------------------------------------
# text processing

def remove_tags(html):

    # parse html content
    soup = BeautifulSoup(html, "html.parser")

    for data in soup(['style', 'script']):
        # Remove tags
        data.decompose()

    # return data by retrieving the tag content
    return ' '.join(soup.stripped_strings)

def remove_emoji(data):
    emoj = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
        u"\U00002500-\U00002BEF"  # chinese char
        u"\U00002702-\U000027B0"
        u"\U000024C2-\U0001F251"
        u"\U0001f926-\U0001f937"
        u"\U00010000-\U0010ffff"
        u"\u2640-\u2642"
        u"\u2600-\u2B55"
        u"\u200d"
        u"\u23cf"
        u"\u23e9"
        u"\u231a"
        u"\ufe0f"  # dingbats
        u"\u3030"
                      "]+", re.UNICODE)
    return re.sub(emoj, '', data)

#-------------------------------------

def get_page(link):
  try:
    #print(link)
    x = requests.get(link)
    raw_html = x.text
    clean_text = remove_tags(raw_html)[:1050]
    clean_text = remove_emoji(clean_text)
    return clean_text
  except:
    print(link)

def get_documents(links):
    pre_processed_text = [get_page(link) for link in links]
    while(None in pre_processed_text):
        pre_processed_text.remove(None)
    pre_processed_text = [i for i in pre_processed_text if len(i) > 999]
    return pre_processed_text

#----------------------------------------

def get_topics(docs):
    # Use llama.cpp to load in a Quantized LLM
    llm = Llama(model_path="openhermes-2.5-mistral-7b.Q4_K_M.gguf", n_gpu_layers=-1, n_ctx=4096, stop=["Q:", "\n"])
    
    prompt = """ Q:
    I have a topic that contains the following documents:
    [DOCUMENTS]
    
    The topic is described by the following keywords: '[KEYWORDS]'.
    
    Based on the above information, can you give a short label of the topic of at most 5 words?
    A:
    """
    
    representation_model = {
    "KeyBERT": KeyBERTInspired(),
    "LLM": LlamaCPP(llm, prompt=prompt),
    }
    
    # Pre-calculate embeddings
    embedding_model = SentenceTransformer("BAAI/bge-small-en")
    embeddings = embedding_model.encode(docs, show_progress_bar=True)
    
    # Pre-reduce embeddings for visualization purposes
    reduced_embeddings = UMAP(n_neighbors=15, n_components=2, min_dist=0.0, metric='cosine', random_state=42).fit_transform(embeddings)
    
    # Define sub-models
    umap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', random_state=42)
    hdbscan_model = HDBSCAN(min_cluster_size=2, metric='euclidean', cluster_selection_method='eom', prediction_data=True)
    
    topic_model = BERTopic(
    
      # Sub-models
      embedding_model=embedding_model,
      umap_model=umap_model,
      hdbscan_model=hdbscan_model,
      representation_model=representation_model,
    
      # Hyperparameters
      top_n_words=10,
      verbose=True
    )
    
    # Train model
    topics, probs = topic_model.fit_transform(docs, embeddings)

    return topic_model

#-------------------------------
# Visualize Topics
def get_figure(topic_model):
    # Prepare logo
    bertopic_logo_response = requests.get(
        "https://raw.githubusercontent.com/MaartenGr/BERTopic/master/images/logo.png",
        stream=True,
        headers={'User-Agent': 'My User Agent 1.0'}
    )
    bertopic_logo = np.asarray(PIL.Image.open(bertopic_logo_response.raw))

    # Create a label for each document
    llm_labels = [re.sub(r'\W+', ' ', label[0][0].split("\n")[0].replace('"', '')) for label in topic_model.get_topics(full=True)["LLM"].values()]
    llm_labels = [label if label else "Unlabelled" for label in llm_labels]
    all_labels = [llm_labels[topic+topic_model._outliers] if topic != -1 else "Unlabelled" for topic in topics]
    
    # Run the visualization
    fig = datamapplot.create_plot(
        reduced_embeddings,
        all_labels,
        label_font_size=11,
        title="ArXiv - BERTopic",
        sub_title="Topics labeled with `openhermes-2.5-mistral-7b`",
        label_wrap_width=20,
        use_medoids=True,
        logo=bertopic_logo,
        logo_width=0.16
    )

    return fig