pages/clustering.py

from collections import defaultdict
import streamlit as st
from utils import load_and_preprocess_data
import pandas as pd
import numpy as np
import altair as alt
from sklearn.mixture import GaussianMixture
import plotly.express as px
import itertools
from typing import Dict, List, Tuple


SIDEBAR_DESCRIPTION = """
# Client clustering

To cluster a client, we adopt the RFM metrics. They stand for:

- R = recency, that is the number of days since the last purchase
    in the store
- F = frequency, that is the number of times a customer has ordered something
- M = monetary value, that is how much a customer has spent buying
    from your business.

Given these 3 metrics, we can cluster the customers and find a suitable
"definition" based on the clusters they belong to. Since the dataset
we're using right now has about 5000 distinct customers, we identify
3 clusters for each metric.

## How we compute the clusters

We resort to a GaussianMixture algorithm. We can think of GaussianMixture
as generalized k-means clustering that incorporates information about
the covariance structure of the data as well as the centers of the clusters.
""".lstrip()

FREQUENCY_CLUSTERS_EXPLAIN = """
The **frequency** denotes how frequently a customer has ordered.

There 3 available clusters for this metric:

- cluster 1: denotes a customer that purchases one or few times (range [{}, {}])
- cluster 2: these customer have a discrete amount of orders (range [{}, {}])
- cluster 3: these customer purchases lots of times (range [{}, {}])

-------
""".lstrip()

RECENCY_CLUSTERS_EXPLAIN = """
The **recency** refers to how recently a customer has bought;

There 3 available clusters for this metric:

- cluster 1: the last order of these client is long time ago (range [{}, {}])
- cluster 2: these are clients that purchases something not very recently (range [{}, {}])
- cluster 3: the last order of these client is a few days/weeks ago (range [{}, {}])

-------
""".lstrip()

MONETARY_CLUSTERS_EXPLAIN = """
The **revenue** refers to how much a customer has spent buying
from your business.

There 3 available clusters for this metric:

- cluster 1: these clients spent little money (range [{}, {}])
- cluster 2: these clients spent a considerable amount of money (range [{}, {}])
- cluster 3: these clients spent lots of money (range [{}, {}])

-------
""".lstrip()

EXPLANATION_DICT = {
    "Frequency_cluster": FREQUENCY_CLUSTERS_EXPLAIN,
    "Recency_cluster": RECENCY_CLUSTERS_EXPLAIN,
    "Revenue_cluster": MONETARY_CLUSTERS_EXPLAIN,
}


def create_features(df: pd.DataFrame):
    """Creates a new dataframe with the RFM features for each client."""
    # Compute frequency, the number of distinct time a user purchased.
    client_features = df.groupby("CustomerID")["InvoiceDate"].nunique().reset_index()
    client_features.columns = ["CustomerID", "Frequency"]

    # Add monetary value, the total revenue for  each single user.
    client_takings = df.groupby("CustomerID")["Price"].sum()
    client_features["Revenue"] = client_takings.values

    # Add recency, i.e. the days since the last purchase in the store.
    max_date = df.groupby("CustomerID")["InvoiceDate"].max().reset_index()
    max_date.columns = ["CustomerID", "LastPurchaseDate"]

    client_features["Recency"] = (
        max_date["LastPurchaseDate"].max() - max_date["LastPurchaseDate"]
    ).dt.days

    return client_features


@st.cache
def cluster_clients(df: pd.DataFrame):
    """Computes the RFM features and clusters for each user based on the RFM metrics."""

    df_rfm = create_features(df)

    for to_cluster, order in zip(
        ["Revenue", "Frequency", "Recency"], ["ascending", "ascending", "descending"]
    ):
        kmeans = GaussianMixture(n_components=3, random_state=42)
        labels = kmeans.fit_predict(df_rfm[[to_cluster]])
        df_rfm[f"{to_cluster}_cluster"] = _order_cluster(kmeans, labels, order)

    return df_rfm


def _order_cluster(cluster_model: GaussianMixture, clusters, order="ascending"):
    """Orders the cluster by `order`."""
    centroids = cluster_model.means_.sum(axis=1)

    if order.lower() == "descending":
        centroids *= -1

    ascending_order = np.argsort(centroids)
    lookup_table = np.zeros_like(ascending_order)
    # Cluster will start from 1
    lookup_table[ascending_order] = np.arange(cluster_model.n_components) + 1
    return lookup_table[clusters]


def show_purhcase_history(user: int, df: pd.DataFrame):
    user_purchases = df.loc[df.CustomerID == user, ["Price", "InvoiceDate"]]
    expenses = user_purchases.groupby(user_purchases.InvoiceDate).sum()
    expenses.columns = ["Expenses"]
    expenses = expenses.reset_index()

    c = (
        alt.Chart(expenses)
        .mark_line(point=True)
        .encode(
            x=alt.X("InvoiceDate", timeUnit="yearmonthdate", title="Date"),
            y="Expenses",
        )
        .properties(title="User expenses")
    )

    st.altair_chart(c)


def show_user_info(user: int, df_rfm: pd.DataFrame):
    """Prints some information about the user.

    The main information are the total expenses, how
    many times he purchases in the store, and the clusters
    he belongs to.
    """

    user_row = df_rfm[df_rfm["CustomerID"] == user]
    if len(user_row) == 0:
        st.write(f"No user with id {user}")

    output = []

    output.append(f"The user purchased **{user_row['Frequency'].squeeze()} times**.\n")
    output.append(
        f"She/he spent **{user_row['Revenue'].squeeze()} dollars** in total.\n"
    )
    output.append(
        f"The last time she/he bought something was **{user_row['Recency'].squeeze()} days ago**.\n"
    )
    output.append(f"She/he belongs to the clusters: ")
    for cluster in [column for column in user_row.columns if "_cluster" in column]:
        output.append(f"- {cluster} = {user_row[cluster].squeeze()}")

    st.write("\n".join(output))

    return (
        user_row["Recency_cluster"].squeeze(),
        user_row["Frequency_cluster"].squeeze(),
        user_row["Revenue_cluster"].squeeze(),
    )


def explain_cluster(cluster_info):
    """Displays a popup menu explinging the meanining of the clusters."""

    with st.expander("Show information about the clusters"):
        st.write(
            "**Note**: these values are valid for these dataset."
            "Different dataset will have different number of clusters"
            " and values"
        )
        for cluster, info in cluster_info.items():
            # Transform the (mins, maxs) tuple into
            # [min_1, max_1, min_2, max_2, ...] list.
            min_max_interleaved = list(itertools.chain(*zip(info[0], info[1])))
            st.write(EXPLANATION_DICT[cluster].format(*min_max_interleaved))


def categorize_user(recency_cluster, frequency_cluster, monetary_cluster):
    """Describe the user with few words based on the cluster he belongs to."""

    score = f"{recency_cluster}{frequency_cluster}{monetary_cluster}"

    # @fixme: find a better approeach. These elif chains don't scale at all.

    description = ""

    if score == "111":
        description = "Tourist"
    elif score.startswith("2"):
        description = "Losing interest"
    elif score == "133":
        description = "Former lover"
    elif score == "123":
        description = "Former passionate client"
    elif score == "113":
        description = "Spent a lot, but never come back"
    elif score.startswith("1"):
        description = "About to dump"
    elif score == "313":
        description = "Potential lover"
    elif score == "312":
        description = "Interesting new client"
    elif score == "311":
        description = "New customer"
    elif score == "333":
        description = "Gold client"
    elif score == "322":
        description = "Lovers"
    else:
        description = "Average client"

    st.write(f"The customer can be described as: **{description}**")


def plot_rfm_distribution(
    df_rfm: pd.DataFrame, cluster_info: Dict[str, Tuple[List[int], List[int]]]
):
    """Plots 3 histograms for the RFM metrics."""

    for x, to_reverse in zip(("Revenue", "Frequency", "Recency"), (False, False, True)):
        fig = px.histogram(
            df_rfm,
            x=x,
            log_y=True,
            title=f"{x} metric",
        )
        # Get the max value in the cluster info. The cluster_info_dict is a
        # tuple with first element the min values of the cluster, and second
        # element the max values of the cluster.
        values = cluster_info[f"{x}_cluster"][1]  # get max values
        print(values)
        # Add vertical bar on each cluster end. But skip the last cluster.
        loop_range = range(len(values) - 1)
        if to_reverse:
            # Skip the last element
            loop_range = range(len(values) - 1, 0, -1)
        for n_cluster in loop_range:
            print(x)
            print(values[n_cluster])
            fig.add_vline(
                x=values[n_cluster],
                annotation_text=f"End of cluster {n_cluster+1}",
                line_dash="dot",
                annotation=dict(textangle=90, font_color="red"),
            )

        fig.update_layout(
            yaxis_title="Count (log scale)",
        )

        st.plotly_chart(fig)


def display_dataframe_heatmap(df_rfm: pd.DataFrame, cluster_info_dict):
    """Displays an heatmap of how many clients lay in the clusters.

    This method uses some black magic coming from the dataframe
    styling guide.
    """

    def style_with_limits(x, column, cluster_limit_dict):
        """Simple function to transform the cluster number into
        a cluster + range string."""
        min_v = cluster_limit_dict[column][0][x - 1]
        max_v = cluster_limit_dict[column][1][x - 1]
        return f"{x}: [{int(min_v)}, {int(max_v)}]"

    # Create a dataframe with the count of clients for each group
    # of cluster.

    count = (
        df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])[
            "CustomerID"
        ]
        .count()
        .reset_index()
    )
    count = count.rename(columns={"CustomerID": "Count"})

    # Remove duplicates
    count = count.drop_duplicates(
        ["Revenue_cluster", "Frequency_cluster", "Recency_cluster"]
    )

    # Add limits to the cells. In this way, we can better display
    # the heatmap.
    for cluster in ["Revenue_cluster", "Frequency_cluster", "Recency_cluster"]:
        count[cluster] = count[cluster].apply(
            lambda x: style_with_limits(x, cluster, cluster_info_dict)
        )

    # Use the count column as values, then index with the clusters.
    count = count.pivot(
        index=["Revenue_cluster", "Frequency_cluster"],
        columns="Recency_cluster",
        values="Count",
    )

    # Style manipulation
    cell_hover = {
        "selector": "td",
        "props": "font-size:1.2em",
    }
    index_names = {
        "selector": ".index_name",
        "props": "font-style: italic; color: Black; font-weight:normal;font-size:1.2em;",
    }
    headers = {
        "selector": "th:not(.index_name)",
        "props": "background-color: White; color: black; font-size:1.2em",
    }

    # Finally, display
    # We cannot directly print the dataframe since the streamlit
    # functin remove the multiindex. Thus, we extract the html representation
    # and then display it.
    st.markdown("## Heatmap: how the client are distributed between clusters")
    st.write(
        count.style.format(thousands=" ", precision=0, na_rep="0")
        .set_table_styles([cell_hover, index_names, headers])
        .background_gradient(cmap="coolwarm")
        .to_html(),
        unsafe_allow_html=True,
    )


def main():
    st.sidebar.markdown(SIDEBAR_DESCRIPTION)

    df, _, _ = load_and_preprocess_data()
    df_rfm = cluster_clients(df)

    st.markdown(
        "# Dataset "
        "\nThis is the processed dataset with information about the clients, such as"
        " the RFM values and the clusters they belong to."
    )
    st.dataframe(df_rfm.style.format(formatter={"Revenue": "{:.2f}"}))

    cluster_info_dict = defaultdict(list)

    with st.expander("Show more details about the clusters"):
        for cluster in [column for column in df_rfm.columns if "_cluster" in column]:
            st.write(cluster)
            cluster_info = (
                df_rfm.groupby(cluster)[cluster.split("_")[0]]
                .describe()
                .reset_index(names="Cluster")
            )
            min_cluster = cluster_info["min"].astype(int)
            max_cluster = cluster_info["max"].astype(int)
            cluster_info_dict[cluster] = (min_cluster, max_cluster)
            st.dataframe(cluster_info)

    st.markdown("## RFM metric distribution")

    plot_rfm_distribution(df_rfm, cluster_info_dict)

    display_dataframe_heatmap(df_rfm, cluster_info_dict)

    st.markdown("## Interactive exploration")

    filter_by_cluster = st.checkbox(
        "Filter client: only one client per cluster type",
        value=True,
    )

    client_to_select = (
        df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])[
            "CustomerID"
        ]
        .first()
        .values
        if filter_by_cluster
        else df["CustomerID"].unique()
    )

    # Let the user select the user to investigate
    user = st.selectbox(
        "Select a customer to show more information about him.",
        client_to_select,
    )

    show_purhcase_history(user, df)

    recency, frequency, revenue = show_user_info(user, df_rfm)

    categorize_user(recency, frequency, revenue)

    explain_cluster(cluster_info_dict)


main()