add init py

RECODE_speckle_utils

@@ -1 +0,0 @@
-Subproject commit 9d8b034751522ec0220f9dfffd702292abce8de4

speckleUtils/color_maps.py

@@ -0,0 +1,36 @@
+def gh_color_blueRed():
+    # grasshoper color scheme 
+    color_list = [[15,16,115],
+            [177,198,242],
+            [251,244,121],
+            [222,140,61],
+            [183,60,34]]
+    # Scale RGB values to [0,1] range
+    color_list = [[c/255. for c in color] for color in color_list]
+    return color_list
+
+def gh_color_whiteRed():
+    # grasshoper color scheme 
+    color_list = [[255,255,255],
+            [111,19,12],
+            ]
+    # Scale RGB values to [0,1] range
+    color_list = [[c/255. for c in color] for color in color_list]
+    return color_list
+
+def gh_color_cluster():
+    # grasshoper color scheme 
+    color_list = [
+            [181,200,230],
+            [227,170,170],
+            [200,200,200],
+            [250,200,254],
+            [200,180,220],
+            [180,220,170],
+            ]
+    # Scale RGB values to [0,1] range
+    color_list = [[c/255. for c in color] for color in color_list]
+    return color_list
+
+#---
+#---

speckleUtils/data_utils.py

@@ -0,0 +1,616 @@
+
+import pandas as pd
+import numpy as np
+import copy
+import os 
+import csv
+import io
+import json
+import requests
+
+try:
+    from fuzzywuzzy import fuzz
+except:
+    pass
+
+def helper():
+    """
+    Prints out the help message for this module.
+    """
+    print("This module contains a set of utility functions for data processing.")
+    print("______________________________________________________________________")
+    print("for detailed help call >>> help(speckle_utils.function_name) <<< ")
+    print("______________________________________________________________________")
+    print("available functions:")
+    print("cleanData(data, mode='drop', num_only=False) -> clean dataframes, series or numpy arrays" )
+    print( """ sort_and_match_df(A, B, uuid_column) -> merges two dataframes by a common uuid comon (best practice: always use this)""")
+    print("transform_to_score(data, minPts, maxPts, t_low, t_high, cull_invalid=False) -> transform data to a score based on percentiles and provided points") 
+    print("colab_create_directory(base_name) -> create a directory with the given name, if it already exists, add a number to the end of the name, usefull for colab")
+    print("colab_zip_download_folder(dir_name) -> zips and downloads a directory from colab. will only work in google colaboratory ")
+
+
+def cleanData(data, mode="drop", num_only=False, print_report=True):
+    """
+    Cleans data by handling missing or null values according to the specified mode.
+
+    Args:
+        data (numpy.ndarray, pandas.DataFrame, pandas.Series): Input data to be cleaned.
+        mode (str, optional): Specifies the method to handle missing or null values.
+                              "drop" drops rows with missing values (default),
+                              "replace_zero" replaces missing values with zero,
+                              "replace_mean" replaces missing values with the mean of the column.
+        num_only (bool, optional): If True and data is a DataFrame, only numeric columns are kept. Defaults to False.#
+        print_report (bool, optional): if True the report is printed to the console. Defaults to True.
+
+    Returns:
+        numpy.ndarray, pandas.DataFrame, pandas.Series: Cleaned data with the same type as the input.
+
+
+    Raises:
+        ValueError: If the input data type is not supported (must be numpy.ndarray, pandas.DataFrame or pandas.Series).
+
+    This function checks the type of the input data and applies the appropriate cleaning operation accordingly.
+    It supports pandas DataFrame, pandas Series, and numpy array. For pandas DataFrame, it can optionally 
+    convert and keep only numeric columns.
+    """
+    report = {}
+    if isinstance(data, pd.DataFrame):
+        initial_cols = data.columns.tolist()
+        initial_rows = data.shape[0]
+        if num_only:
+            # attempt casting before doing this selection
+            data = data.apply(pd.to_numeric, errors='coerce')
+            data = data.select_dtypes(include=['int64', 'float64'])
+            report['dropped_cols'] = list(set(initial_cols) - set(data.columns.tolist()))
+
+        if mode == "drop":
+            data = data.dropna()
+            report['dropped_rows'] = initial_rows - data.shape[0]
+        elif mode=="replace_zero":
+            data = data.fillna(0)
+        elif mode=="replace_mean":
+            data = data.fillna(data.mean())
+
+    elif isinstance(data, pd.Series):
+        initial_length = len(data)
+        if mode == "drop":
+            data = data.dropna()
+            report['dropped_rows'] = initial_length - len(data)
+        elif mode=="replace_zero":
+            data = data.fillna(0)
+        elif mode=="replace_mean":
+            data = data.fillna(data.mean())
+
+    elif isinstance(data, np.ndarray):
+        initial_length = data.size
+        if mode=="drop":
+            data = data[~np.isnan(data)]
+            report['dropped_rows'] = initial_length - data.size
+        elif mode=="replace_zero":
+            data = np.nan_to_num(data, nan=0)
+        elif mode=="replace_mean":
+            data = np.where(np.isnan(data), np.nanmean(data), data)
+
+    else:
+        raise ValueError("Unsupported data type")
+    if print_report:
+        print(report)   
+    return data
+
+
+
+def sort_and_match_df(A, B, uuid_column):
+    """
+    Sorts and matches DataFrame B to A based on a shared uuid_column.
+    Prioritizes uuid_column as an index if present, otherwise uses it as a column.
+    
+    Parameters:
+    A, B (DataFrame): Input DataFrames to be sorted and matched.
+    uuid_column (str): Shared column/index for matching rows.
+    
+    Returns:
+    DataFrame: Resulting DataFrame after left join of A and B on uuid_column.
+    """
+    if uuid_column in A.columns:
+        A = A.set_index(uuid_column, drop=False)
+    if uuid_column in B.columns:
+        B = B.set_index(uuid_column, drop=False)
+
+    merged_df = pd.merge(A, B, left_index=True, right_index=True, how='left')
+    return merged_df.reset_index(drop=False)
+
+
+def sort_and_match_dfs(dfs, uuid_column):
+    """
+    Sorts and matches all DataFrames in list based on a shared uuid_column.
+    Prioritizes uuid_column as an index if present, otherwise uses it as a column.
+    Raises a warning if any two DataFrames have overlapping column names.
+    
+    Parameters:
+    dfs (list): A list of DataFrames to be sorted and matched.
+    uuid_column (str): Shared column/index for matching rows.
+    
+    Returns:
+    DataFrame: Resulting DataFrame after successive left joins on uuid_column.
+    """
+    if not dfs:
+        raise ValueError("The input list of DataFrames is empty")
+    
+    # Convert uuid_column to index if it's a column
+    for i, df in enumerate(dfs):
+        if uuid_column in df.columns:
+            dfs[i] = df.set_index(uuid_column, drop=False)
+
+    # Check for overlapping column names
+    all_columns = [set(df.columns) for df in dfs]
+    for i, columns_i in enumerate(all_columns):
+        for j, columns_j in enumerate(all_columns[i+1:], start=i+1):
+            overlapping_columns = columns_i.intersection(columns_j) - {uuid_column}
+            if overlapping_columns:
+                print(f"Warning: DataFrames at indices {i} and {j} have overlapping column(s): {', '.join(overlapping_columns)}")
+    
+    result_df = dfs[0]
+    for df in dfs[1:]:
+        result_df = pd.merge(result_df, df, left_index=True, right_index=True, how='left')
+
+    return result_df.reset_index(drop=False)
+
+
+
+
+def transform_to_score(data, minPts, maxPts, t_low, t_high, cull_invalid=False):
+    """
+        Transforms data to a score based on percentiles and provided points.
+
+        Args:
+            data (numpy.array or pandas.Series): Input data to be transformed.
+            minPts (float): The minimum points to be assigned.
+            maxPts (float): The maximum points to be assigned.
+            t_low (float): The lower percentile threshold.
+            t_high (float): The upper percentile threshold.
+            cull_invalid (bool, optional): If True, invalid data is removed. Defaults to False.
+
+        Returns:
+            numpy.array: The transformed data, where each element has been converted to a score based on its percentile rank.
+
+        This function calculates the t_low and t_high percentiles of the input data, and uses linear interpolation
+        to transform each data point to a score between minPts and maxPts. Any data point that falls above the t_high
+        percentile is given a score of maxPts. If cull_invalid is True, any invalid data points (such as NaNs or 
+        infinite values) are removed before the transformation is applied.
+    """
+
+    # If cull_invalid is True, the data is cleaned and invalid data is removed.
+    if cull_invalid:
+        inp_data = cleanData(inp_data, mode="drop", num_only=True)
+    
+    # Calculate the percentile values based on the data
+    percentile_low = np.percentile(data, t_low)
+    percentile_high = np.percentile(data, t_high)
+
+    # Create a copy of the data to store the transformed points
+    transformed_data = data.copy()
+
+    # Apply linear interpolation between minPts and maxPts
+    transformed_data = np.interp(transformed_data, [percentile_low, percentile_high], [minPts, maxPts])
+
+    # Replace values above the percentile threshold with maxPts
+    transformed_data[transformed_data >= percentile_high] = maxPts
+
+    return transformed_data
+
+
+def colab_create_directory(base_name):
+    """ creates a directory with the given name, if it already exists, add a number to the end of the name.
+    Usefull for colab to batch save e.g. images and avoid overwriting.
+    Args:
+        base_name (str): name of the directory to create
+    Returns:
+        str: name of the created directory"""
+    counter = 1
+    dir_name = base_name
+
+    while os.path.exists(dir_name):
+        dir_name = f"{base_name}_{counter}"
+        counter += 1
+
+    os.mkdir(dir_name)
+    return dir_name
+
+def smart_round(x):
+    if abs(x) >= 1000:
+        return round(x)
+    elif abs(x) >= 10:
+        return round(x, 1)
+    elif abs(x) >= 1:
+        return round(x, 2)
+    else:
+        return round(x, 3)
+
+def colab_zip_download_folder(dir_name):
+    """ zips and downloads a directory from colab. will only work in google colab
+    Args: 
+        dir_name (str): name of the directory to zip and download
+    returns: 
+        None, file will be downloaded to the local machine"""
+    try:
+        # zip the directory
+        get_ipython().system('zip -r /content/{dir_name}.zip /content/{dir_name}')
+
+        # download the zip file
+        from google.colab import files
+        files.download(f"/content/{dir_name}.zip")
+    except:
+        print("something went wrong, this function will only work in google colab, make sure to import the necessary packages. >>> from google.colab import files <<<" ) 
+
+
+
+
+def generate__cluster_prompt(data_context, analysis_goal, column_descriptions, cluster_stat, complexity, exemplary_cluster_names_descriptions=None, creativity=None):
+    # Define complexity levels
+    complexity_levels = {
+        1: "Please explain the findings in a simple way, suitable for someone with no knowledge of statistics or data science.",
+        2: "Please explain the findings in moderate detail, suitable for someone with basic understanding of statistics or data science.",
+        3: "Please explain the findings in great detail, suitable for someone with advanced understanding of statistics or data science."
+    }
+
+    # Start the prompt
+    prompt = f"The data you are analyzing is from the following context: {data_context}. The goal of this analysis is: {analysis_goal}.\n\n"
+
+    # Add column descriptions
+    prompt += "The data consists of the following columns:\n"
+    for column, description in column_descriptions.items():
+        prompt += f"- {column}: {description}\n"
+
+    # Add cluster stat and ask for generation
+    prompt += "\nBased on the data, the following cluster has been identified:\n"
+    prompt += f"\nCluster ID: {cluster_stat['cluster_id']}\n"
+    for column, stats in cluster_stat['columns'].items():
+        prompt += f"- {column}:\n"
+        for stat, value in stats.items():
+            prompt += f"  - {stat}: {value}\n"
+    
+    # Adjust the prompt based on whether examples are provided
+    if exemplary_cluster_names_descriptions is not None and creativity is not None:
+        prompt += f"\nPlease generate a name and description for this cluster, using a creativity level of {creativity} (where 0 is sticking closely to the examples and 1 is completely original). The examples provided are: {exemplary_cluster_names_descriptions}\n"
+    else:
+        prompt += "\nPlease generate a name and description for this cluster. Be creative and original in your descriptions.\n"
+
+    prompt += "Please fill the following JSON template with the cluster name and two types of descriptions:\n"
+    prompt += "{\n  \"cluster_name\": \"<generate>\",\n  \"description_narrative\": \"<generate>\",\n  \"description_statistical\": \"<generate>\"\n}\n"
+    prompt += f"\nFor the narrative description, {complexity_levels[complexity]}"
+
+    return prompt
+
+
+def generate_cluster_description(cluster_df, original_df=None, stats_list=['mean', 'min', 'max', 'std', 'kurt'], cluster_id = ""):
+    cluster_description = {"cluster_id": cluster_id,
+                            "name":"<generate>",
+                              "description_narrative":"<generate>",
+                              "description_statistical":"<generate>",
+                              "size": len(cluster_df),
+                                "columns": {}
+                                }
+    if original_df is not None:
+        size_relative = round(len(cluster_df)/len(original_df), 2)
+    for column in cluster_df.columns:
+        cluster_description["columns"][column] = {}
+        for stat in stats_list:
+            # Compute the statistic for the cluster
+            if stat == 'mean':
+                value = round(cluster_df[column].mean(),2)
+            elif stat == 'min':
+                value = round(cluster_df[column].min(),2)
+            elif stat == 'max':
+                value = round(cluster_df[column].max(),2)
+            elif stat == 'std':
+                value = round(cluster_df[column].std(), 2)
+            elif stat == 'kurt':
+                value = round(cluster_df[column].kurt(), 2)
+
+            # Compute the relative difference if the original dataframe is provided
+            if original_df is not None:
+                original_value = original_df[column].mean() if stat == 'mean' else original_df[column].min() if stat == 'min' else original_df[column].max() if stat == 'max' else original_df[column].std() if stat == 'std' else original_df[column].kurt()
+                relative_difference = (value - original_value) / original_value * 100
+                cluster_description["columns"][column][stat] = {"value": round(value,2), "relative_difference": f"{round(relative_difference,2)}%"}
+            else:
+                cluster_description["columns"][column][stat] = {"value": round(value,2)}
+
+    return cluster_description
+
+
+
+
+def generate_cluster_description_mixed(cluster_df, original_df=None, stats_list=['mean', 'min', 'max', 'std', 'kurt'], cluster_id = ""):
+    cluster_description = {
+        "cluster_id": cluster_id,
+        "name":"<generate>",
+        "description_narrative":"<generate>",
+        "description_statistical":"<generate>",
+        "size": len(cluster_df),
+        "columns": {}
+    }
+
+    if original_df is not None:
+        size_relative = round(len(cluster_df)/len(original_df), 2)
+
+    # Create CSV string in memory
+    csv_io = io.StringIO()
+    writer = csv.writer(csv_io)
+
+    # CSV Headers
+    writer.writerow(['Column', 'Stat', 'Value', 'Relative_Difference'])
+
+    for column in cluster_df.columns:
+        for stat in stats_list:
+            if stat == 'mean':
+                value = round(cluster_df[column].mean(),2)
+            elif stat == 'min':
+                value = round(cluster_df[column].min(),2)
+            elif stat == 'max':
+                value = round(cluster_df[column].max(),2)
+            elif stat == 'std':
+                value = round(cluster_df[column].std(), 2)
+            elif stat == 'kurt':
+                value = round(cluster_df[column].kurt(), 2)
+
+            if original_df is not None:
+                original_value = original_df[column].mean() if stat == 'mean' else original_df[column].min() if stat == 'min' else original_df[column].max() if stat == 'max' else original_df[column].std() if stat == 'std' else original_df[column].kurt()
+                relative_difference = (value - original_value) / original_value * 100
+                writer.writerow([column, stat, value, f"{round(relative_difference,2)}%"])
+            else:
+                writer.writerow([column, stat, value, "N/A"])
+
+    # Store CSV data in JSON
+    cluster_description["columns"] = csv_io.getvalue()
+
+    data_description = """
+        The input data is a JSON object with details about clusters. It has the following structure:
+
+        1. 'cluster_id': An identifier for the cluster.
+        2. 'name': A placeholder for the name of the cluster.
+        3. 'description_narrative': A placeholder for a narrative description of the cluster.
+        4. 'description_statistical': A placeholder for a statistical description of the cluster.
+        5. 'size': The number of elements in the cluster.
+        6. 'columns': This contains statistical data about different aspects, presented in CSV format. 
+
+        In the 'columns' CSV:
+        - 'Column' corresponds to the aspect.
+        - 'Stat' corresponds to the computed statistic for that aspect in the cluster.
+        - 'Value' is the value of that statistic.
+        - 'Relative_Difference' is the difference of the statistic's value compared to the average value of this statistic in the entire dataset, expressed in percentages.
+        """
+
+    return cluster_description, data_description
+
+# ==================================================================================================
+# ========== TESTING ===============================================================================
+
+def compare_column_names(ref_list, check_list):
+    """
+    Compares two lists of column names to check for inconsistencies.
+
+    Args:
+        ref_list (list): The reference list of column names.
+        check_list (list): The list of column names to be checked.
+
+    Returns:
+        report_dict (dict): Report about the comparison process.
+
+    Raises:
+        ValueError: If the input types are not list.
+    """
+    # Check the type of input data
+    if not all(isinstance(i, list) for i in [ref_list, check_list]):
+        raise ValueError("Both inputs must be of type list")
+
+    missing_cols = [col for col in ref_list if col not in check_list]
+    extra_cols = [col for col in check_list if col not in ref_list]
+    
+    try:
+      typos = {}
+      for col in check_list:
+          if col not in ref_list:
+              similarity_scores = {ref_col: fuzz.ratio(col, ref_col) for ref_col in ref_list}
+              likely_match = max(similarity_scores, key=similarity_scores.get)
+              if similarity_scores[likely_match] > 70:  # you may adjust this threshold as needed
+                  typos[col] = likely_match
+    except:
+      typos = {"error":"fuzzywuzzy is probably not installed"}
+
+    report_dict = {
+        "missing_columns": missing_cols,
+        "extra_columns": extra_cols,
+        "likely_typos": typos
+    }
+
+    print("\nREPORT:")
+    print('-'*50)
+    print("\n- Missing columns:")
+    print('   ' + '\n   '.join(f'"{col}"' for col in missing_cols) if missing_cols else '   None')
+    print("\n- Extra columns:")
+    print('   ' + '\n   '.join(f'"{col}"' for col in extra_cols) if extra_cols else '   None')
+    print("\n- Likely typos:")
+    if typos:
+        for k, v in typos.items():
+            print(f'   "{k}": "{v}"')
+    else:
+        print('   None')
+
+    return report_dict
+
+
+def compare_dataframes(df1, df2, threshold=0.1):
+    """
+    Compare two pandas DataFrame and returns a report highlighting any significant differences.
+    Significant differences are defined as differences that exceed the specified threshold.
+
+    Args:
+        df1, df2 (pandas.DataFrame): Input dataframes to be compared.
+        threshold (float): The percentage difference to be considered significant. Defaults to 0.1 (10%).
+
+    Returns:
+        pandas.DataFrame: A report highlighting the differences between df1 and df2.
+    """
+    # Column comparison
+    cols_df1 = set(df1.columns)
+    cols_df2 = set(df2.columns)
+    
+    common_cols = cols_df1 & cols_df2
+    missing_df1 = cols_df2 - cols_df1
+    missing_df2 = cols_df1 - cols_df2
+
+    print("Column Comparison:")
+    print("------------------")
+    print(f"Common columns ({len(common_cols)}): {sorted(list(common_cols)) if common_cols else 'None'}")
+    print(f"Columns missing in df1 ({len(missing_df1)}): {sorted(list(missing_df1)) if missing_df1 else 'None'}")
+    print(f"Columns missing in df2 ({len(missing_df2)}): {sorted(list(missing_df2)) if missing_df2 else 'None'}")
+    print("\n")
+
+    # Check for new null values
+    print("Null Values Check:")
+    print("------------------")
+    inconsistent_values_cols = []
+    inconsistent_ranges_cols = []
+    constant_cols = []
+    
+    for col in common_cols:
+        nulls1 = df1[col].isnull().sum()
+        nulls2 = df2[col].isnull().sum()
+        if nulls1 == 0 and nulls2 > 0:
+            print(f"New null values detected in '{col}' of df2.")
+
+        # Check for value consistency
+        if df1[col].nunique() <= 10 and df2[col].nunique() <= 10:
+            inconsistent_values_cols.append(col)
+
+        
+          # Check for range consistency
+        if df1[col].dtype.kind in 'if' and df2[col].dtype.kind in 'if':
+          range1 = df1[col].max() - df1[col].min()
+          range2 = df2[col].max() - df2[col].min()
+          diff = abs(range1 - range2)
+          mean_range = (range1 + range2) / 2
+          if diff / mean_range * 100 > threshold * 100:
+              inconsistent_ranges_cols.append(col)
+
+        # Check for constant columns
+        if len(df1[col].unique()) == 1 or len(df2[col].unique()) == 1:
+            constant_cols.append(col)
+
+    # Print out the results of value consistency, range consistency, and constant columns check
+    print("\nValue Consistency Check:")
+    print("------------------------")
+    print(f"Columns with inconsistent values (checks if the unique values are the same in both dataframes): {inconsistent_values_cols if inconsistent_values_cols else 'None'}")
+    
+    print("\nRange Consistency Check (checks if the range (max - min) of the values in both dataframes is consistent):")
+    print("------------------------")
+    print(f"Columns with inconsistent ranges: {inconsistent_ranges_cols if inconsistent_ranges_cols else 'None'}")
+    
+    print("\nConstant Columns Check (columns that have constant values in either dataframe):")
+    print("-----------------------")
+    print(f"Constant columns: {constant_cols if constant_cols else 'None'}")
+
+    # Check for changes in data type
+    print("\nData Type Check:")
+    print("----------------")
+    for col in common_cols:
+        dtype1 = df1[col].dtype
+        dtype2 = df2[col].dtype
+        if dtype1 != dtype2:
+            print(f"df1 '{dtype1}' -> '{dtype2}' in df2, Data type for '{col}' has changed.")
+    print("\n")
+    
+  
+
+    report_dict = {"column": [], "statistic": [], "df1": [], "df2": [], "diff%": []}
+    statistics = ["mean", "std", "min", "25%", "75%", "max", "nulls", "outliers"]
+
+    for col in common_cols:
+        if df1[col].dtype in ['int64', 'float64'] and df2[col].dtype in ['int64', 'float64']:
+            desc1 = df1[col].describe()
+            desc2 = df2[col].describe()
+            for stat in statistics[:-2]:
+                report_dict["column"].append(col)
+                report_dict["statistic"].append(stat)
+                report_dict["df1"].append(desc1[stat])
+                report_dict["df2"].append(desc2[stat])
+                diff = abs(desc1[stat] - desc2[stat])
+                mean = (desc1[stat] + desc2[stat]) / 2
+                report_dict["diff%"].append(diff / mean * 100 if mean != 0 else 0)  # Fix for division by zero
+            nulls1 = df1[col].isnull().sum()
+            nulls2 = df2[col].isnull().sum()
+            outliers1 = df1[(df1[col] < desc1["25%"] - 1.5 * (desc1["75%"] - desc1["25%"])) | 
+                            (df1[col] > desc1["75%"] + 1.5 * (desc1["75%"] - desc1["25%"]))][col].count()
+            outliers2 = df2[(df2[col] < desc2["25%"] - 1.5 * (desc2["75%"] - desc2["25%"])) | 
+                            (df2[col] > desc2["75%"] + 1.5 * (desc2["75%"] - desc2["25%"]))][col].count()
+            for stat, value1, value2 in zip(statistics[-2:], [nulls1, outliers1], [nulls2, outliers2]):
+                report_dict["column"].append(col)
+                report_dict["statistic"].append(stat)
+                report_dict["df1"].append(value1)
+                report_dict["df2"].append(value2)
+                diff = abs(value1 - value2)
+                mean = (value1 + value2) / 2
+                report_dict["diff%"].append(diff / mean * 100 if mean != 0 else 0)  # Fix for division by zero
+
+    report_df = pd.DataFrame(report_dict)
+    report_df["significant"] = report_df["diff%"] > threshold * 100
+    report_df = report_df[report_df["significant"]]
+    report_df = report_df.round(2)
+
+    print(f"REPORT:\n{'-'*50}")
+    for col in report_df["column"].unique():
+        print(f"\n{'='*50}")
+        print(f"Column: {col}\n{'='*50}")
+        subset = report_df[report_df["column"]==col][["statistic", "df1", "df2", "diff%"]]
+        subset.index = subset["statistic"]
+        print(subset.to_string(header=True))
+
+    return report_df
+
+
+def notion_db_as_df(database_id, token):
+    base_url = "https://api.notion.com/v1"
+
+    # Headers for API requests
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Notion-Version": "2022-06-28",
+        "Content-Type": "application/json"
+    }
+
+    response = requests.post(f"{base_url}/databases/{database_id}/query", headers=headers)
+    # response.raise_for_status()  # Uncomment to raise an exception for HTTP errors
+    pages = response.json().get('results', [])
+    print(response.json().keys())
+
+    # Used to create df
+    table_data = {}
+    page_cnt = len(pages)
+    for i, page in enumerate(pages):
+        for cur_col, val in page["properties"].items():
+            if cur_col not in table_data:
+                table_data[cur_col] = [None] * page_cnt
+            val_type = val["type"]
+            if val_type == "title":
+                value = val[val_type][0]["text"]["content"]
+            elif val_type in ["number", "checkbox"]:
+                value = val[val_type]
+            elif val_type in ["select", "multi_select"]:
+                value = ', '.join([option["name"] for option in val[val_type]])
+            elif val_type == "date":
+                value = val[val_type]["start"]
+            elif val_type in ["people", "files"]:
+                value = ', '.join([item["id"] for item in val[val_type]])
+            elif val_type in ["url", "email", "phone_number"]:
+                value = val[val_type]
+            elif val_type == "formula":
+                value = val[val_type]["string"] if "string" in val[val_type] else val[val_type]["number"]
+            elif val_type == "rich_text":
+              value = val[val_type][0]["text"]["content"]
+            else:
+                value = str(val[val_type])  # Fallback to string representation
+            table_data[cur_col][i] = value
+
+    # To DataFrame
+    df = pd.DataFrame(table_data)
+    return df

speckleUtils/plots_utils.py

@@ -0,0 +1,814 @@
+import json
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+import numpy as np
+import math
+
+import matplotlib.pyplot as plt
+import matplotlib.patches as patches
+import matplotlib.colors as colors
+from matplotlib.colors import ListedColormap, LinearSegmentedColormap, Normalize
+from matplotlib.cm import ScalarMappable
+import pandas as pd
+import numpy as np
+from pandas.api.types import is_numeric_dtype
+from mpl_toolkits.axes_grid1 import make_axes_locatable
+
+from sklearn.metrics import r2_score
+
+def cleanData(data, mode="drop", num_only=False):
+    """
+    This function cleans the input data based on the specified mode.
+
+    Parameters:
+    data (pd.DataFrame, pd.Series, or np.ndarray): The input data to be cleaned.
+    mode (str, optional): The cleaning method, one of "drop", "replace_zero", or "replace_mean". 
+                          "drop" removes NaN values, 
+                          "replace_zero" replaces NaN values with zeros,
+                          "replace_mean" replaces NaN values with the mean of the data.
+                          Defaults to "drop".
+    num_only (bool, optional): If True and data is a DataFrame, only integer and float columns are kept.
+                               Defaults to False.
+
+    Returns:
+    data (same type as input): The cleaned data.
+
+    The function works with pandas DataFrame, Series, and numpy array. Depending on the 'mode' argument, 
+    it either drops the NaN values, replaces them with zero, or replaces them with the mean of the data. 
+    If the data is a DataFrame and num_only is set to True, the function only keeps the columns with 
+    numeric data (int64 and float64 dtypes).
+    """
+    # check the type of input data
+    if isinstance(data, pd.DataFrame):
+        if num_only:
+            data = data.select_dtypes(include=['int64', 'float64'])
+        else:
+          data_copy = data.copy()
+          for col in data.columns:
+              data[col] = pd.to_numeric(data[col], errors='coerce')
+              data[col].fillna(data_copy[col], inplace=True)
+  
+        if mode == "drop":
+            data = data.dropna()
+        elif mode=="replace_zero":
+            data = data.fillna(0)
+        elif mode=="replace_mean":
+            data = data.fillna(data.mean())
+
+    elif isinstance(data, pd.Series):
+        if mode == "drop":
+            data = data.dropna()
+        elif mode=="replace_zero":
+            data = data.fillna(0)
+        elif mode=="replace_mean":
+            data = data.fillna(data.mean())
+
+    elif isinstance(data, np.ndarray):
+        if mode=="drop":
+            data = data[~np.isnan(data)]
+        elif mode=="replace_zero":
+            data = np.nan_to_num(data, nan=0)
+        elif mode=="replace_mean":
+            data = np.where(np.isnan(data), np.nanmean(data), data)
+
+    else:
+        raise ValueError("Unsupported data type")
+
+    return data
+
+def boxPlot(inp_data, columName, cull_invalid=True):
+  """
+    This function generates a boxplot for a given set of data.
+
+    Parameters:
+    inp_data (array or list): Input data for which the boxplot is to be created.
+    columName (str): The name of the column which the data represents, to be used as title for the boxplot.
+    cull_invalid (bool, optional): If True, invalid entries in the data are dropped. Defaults to True.
+    
+    Returns:
+    fig (matplotlib Figure object): Figure containing the boxplot.
+    ax (matplotlib Axes object): Axes of the created boxplot.
+
+    The function creates a boxplot of the provided data, marking the 25th, 50th, and 75th percentiles. 
+    The style of the boxplot is custom, with specific colors and properties for different boxplot elements. 
+    The figure title is set to the provided column name.
+    """
+  if cull_invalid == True:
+    inp_data = cleanData(inp_data, mode="drop", num_only=True)
+
+  # Create a new figure
+  fig, ax = plt.subplots(figsize=(10,3), dpi=200)
+
+  # Set the style to white background
+  sns.set_style("white")
+
+  # Calculate the min, max, Q1, and Q3 of the data
+  min_val = np.min(inp_data)
+  max_val = np.max(inp_data)
+  Q1 = np.percentile(inp_data, 25)
+  Q3 = np.percentile(inp_data, 75)
+  mean_val = np.mean(inp_data)
+
+  # Define the positions and labels for the x ticks
+  x_ticks = [] #[min_val, mean_val, Q3, max_val]
+  x_tick_labels =[] #[ round(v,1) for v in x_ticks]
+
+  # Add vertical lines at mean and Q3
+  ax.vlines([mean_val], ymin=-0.35, ymax=0.35, colors='black', linewidth=3)
+  ax.text(mean_val, -0.35, '  mean', ha='left', va='top',  fontsize=14)
+
+  # Define the properties for the boxplot elements
+  boxprops = {'edgecolor': 'black', 'linewidth': 2, 'facecolor': 'white', 'alpha':0.5}
+  medianprops = {'color': 'gray', 'linewidth': 0}
+  whiskerprops = {'color': 'black', 'linewidth': 1}
+  capprops = {'color': 'black', 'linewidth': 2}
+  flierprops = {'marker':'o', 'markersize':3, 'color':'white',  'markerfacecolor':'lightgray'}
+  meanprops = {'color': 'black', 'linewidth': 1.0}
+  kwargs = {'meanline': True, 'showmeans': True}
+
+  # Create the boxplot
+  bplot = sns.boxplot(x=inp_data, 
+              boxprops=boxprops, 
+              medianprops=medianprops, 
+              whiskerprops=whiskerprops, 
+              capprops=capprops,
+              flierprops=flierprops,
+              meanprops=meanprops,
+              width=0.3,
+              ax=ax,
+              **kwargs
+              )
+
+  # Set the figure title and place it on the top left corner
+  ax.set_title(columName, loc='left', color="lightgrey", alpha =0.2)
+
+  # Remove the black outline from the figure
+  for spine in ax.spines.values():
+      spine.set_visible(False)
+
+  # Set the x-axis ticks and labels
+  ax.set_xticks(x_ticks)
+  ax.set_xticklabels(x_tick_labels)
+
+  # Remove the x-axis label
+  ax.set_xlabel('')
+    
+  return fig, ax
+
+
+
+def boxPlot_colorbar(inp_data, columName, cull_invalid=True, color =  ['blue', 'red']):
+  """
+    This function creates a boxplot with an integrated colorbar for a given set of data. 
+
+    Parameters:
+    inp_data (array or list): Input data for which the boxplot is to be created.
+    columName (str): The name of the column which the data represents, to be used as title for the boxplot.
+    cull_invalid (bool, optional): If True, invalid entries in the data are dropped. Defaults to True.
+    color (list of str, optional): List of colors to use for the gradient colorbar. Defaults to ['blue', 'red'].
+    
+    Returns:
+    fig (matplotlib Figure object): Figure containing the boxplot.
+    ax (matplotlib Axes object): Axes of the created boxplot.
+
+    The function creates a boxplot of the provided data, marking the 25th, 50th, and 75th percentiles. 
+    It also creates a horizontal colorbar above the boxplot that serves as a gradient from the minimum 
+    to the maximum values of the data, emphasizing the data distribution.
+    """
+  if cull_invalid == True:
+    inp_data = cleanData(inp_data, mode="drop", num_only=True)
+
+  # Create a new figure
+    fig, (cax, ax) = plt.subplots(nrows=2, figsize=(10,3), dpi=75,
+        gridspec_kw={'height_ratios': [0.1, 1], 'hspace': 0.02}) # Adjust hspace for less space between plots
+
+
+  # Set the style to white background
+  sns.set_style("white")
+
+  # Calculate the min, max, Q1, and Q3 of the data
+  min_val = np.min(inp_data)
+  max_val = np.max(inp_data)
+  Q1 = np.percentile(inp_data, 25)
+  Q3 = np.percentile(inp_data, 75)
+  mean_val = np.mean(inp_data)
+
+  ratio = int(np.ceil((Q3 - min_val) / (max_val - min_val) * 100))
+
+  # Create a custom colormap
+  cmap1 = LinearSegmentedColormap.from_list("mycmap", color)
+  colors = np.concatenate((cmap1(np.linspace(0, 1, ratio)), np.repeat([cmap1(1.)], 100 - ratio, axis=0)))
+  cmap2 = ListedColormap(colors)
+
+  norm = Normalize(vmin=min_val, vmax=max_val)
+  sm = ScalarMappable(norm=norm, cmap=cmap2)
+
+  # Draw a vertical line at Q3
+  cax.axvline(Q3*0.97, color='k', linewidth=3)
+  cbar = fig.colorbar(sm, cax=cax, orientation='horizontal', ticks=[])
+
+  # Define the positions and labels for the x ticks
+  x_ticks = [] #[min_val, mean_val, Q3, max_val]
+  x_tick_labels =[] #[ round(v,1) for v in x_ticks]
+
+  # Add vertical lines at mean and Q3
+  ax.vlines([Q3], ymin=-0.35, ymax=0.35, colors='black', linewidth=3)
+  ax.text(Q3, 0.83, '  75th percentile', ha='left', va='top', transform=ax.get_xaxis_transform(), fontsize=14)
+
+
+  # Define the properties for the boxplot elements
+  boxprops = {'edgecolor': 'black', 'linewidth': 2, 'facecolor': 'white', 'alpha':0.5}
+  medianprops = {'color': 'gray', 'linewidth': 0}
+  whiskerprops = {'color': 'black', 'linewidth': 1}
+  capprops = {'color': 'black', 'linewidth': 2}
+  flierprops = {'marker':'o', 'markersize':3, 'color':'white',  'markerfacecolor':'lightgray'}
+  meanprops = {'color': 'black', 'linewidth': 1.0}
+  kwargs = {'meanline': True, 'showmeans': True}
+
+  # Create the boxplo
+  bplot = sns.boxplot(x=inp_data, 
+              boxprops=boxprops, 
+              medianprops=medianprops, 
+              whiskerprops=whiskerprops, 
+              capprops=capprops,
+              flierprops=flierprops,
+              meanprops=meanprops,
+              width=0.3,
+              ax=ax,
+              **kwargs
+              )
+
+  # Set the figure title and place it on the top left corner
+  ax.set_title(columName, loc='left', color="lightgrey", alpha=0.2)
+
+  # Remove the black outline from the figure
+  for spine in ax.spines.values():
+      spine.set_visible(False)
+
+  # Set the x-axis ticks and labels
+  ax.set_xticks(x_ticks)
+  ax.set_xticklabels(x_tick_labels)
+
+  # Remove the x-axis label
+  ax.set_xlabel('')
+    
+  return fig, ax
+
+
+
+
+
+
+def histogramScore(inp_data,columName, cull_invalid=True):
+  # Create a new figure
+  if cull_invalid:
+    inp_data = cleanData(inp_data, mode="drop", num_only=True)
+
+  fig, ax = plt.subplots()
+
+  # Set the style to white background
+  sns.set_style("white")
+
+  # Create the histogram with an automatic number of bins
+  ax.hist(inp_data, edgecolor='black', facecolor=(0.99,0.99,0.99,1), bins='auto')
+
+  # Remove the black outline from the figure
+  for spine in ax.spines.values():
+      spine.set_visible(False)
+
+  # Make the y-axis visible
+  ax.spines['left'].set_visible(True)
+  ax.spines['left'].set_color("lightgrey")
+  ax.spines['bottom'].set_visible(True)
+  ax.spines['bottom'].set_color("lightgrey")
+
+  # Calculate the min, max, Q1, and Q3 of the data
+  min_val = np.min(inp_data)
+  max_val = np.max(inp_data)
+  Q1 = np.percentile(inp_data, 25)
+  Q3 = np.percentile(inp_data, 75)
+  mean_val = np.mean(inp_data)
+
+
+
+  # Calculate two equally spaced values on either side of the mean
+  step = (mean_val - min_val) / 2
+  xticks = [mean_val - 2*step, mean_val - step, mean_val, max_val]
+  xticks = [ round(v,1) for v in xticks]
+ 
+  ax.set_xticks(xticks)
+
+  # Add a dotted line at the mean value
+  ax.axvline(x=mean_val, ymax=0.85, linestyle='dotted', color='black')
+
+  # Add a text tag at the end of the line  
+  ax.text(mean_val, ax.get_ylim()[1] * 0.98,"Mean", weight = "bold", size=22, ha="center", 
+            bbox=dict(facecolor='white', edgecolor='white', boxstyle='round,pad=0.2'))
+  ax.text(mean_val, ax.get_ylim()[1] * 0.85, str(round(mean_val,1)) + " from " + str(round(max_val,1)), ha='center', va='bottom', size=22,
+            bbox=dict(facecolor='white', edgecolor='white', boxstyle='round,pad=0.2'))
+
+  # Set the figure title and place it on the top left corner
+  ax.set_title(columName, loc='left', color="lightgrey", alpha=0.3)
+
+  # Make the y-axis tick labels smaller
+  ax.tick_params(axis='y', labelsize=8)
+
+  # Remove the x-axis label
+  ax.set_xlabel('')
+
+
+  return fig, ax
+
+
+# =============================================================================
+#==============================================================================
+
+
+def get_drawing_order(dataset, order_of_importance, sorting_direction):
+    # for activity nodes 
+    temp_dataset = dataset.copy()
+    temp_dataset[['id1', 'id2', 'id3']] = temp_dataset['ids'].str.split(';', expand=True).astype(int)
+    columns_ordered = [f'id{i}' for i in order_of_importance]
+    sorting_direction_ordered = [direction == '+' for direction in sorting_direction]
+    drawing_order = temp_dataset.sort_values(columns_ordered, ascending=sorting_direction_ordered).index.tolist()
+    return drawing_order
+
+
+def calculate_aspect_ratio(all_x_coords, all_y_coords):
+    x_range = max(all_x_coords) - min(all_x_coords)
+    y_range = max(all_y_coords) - min(all_y_coords)
+    aspect_ratio = y_range / x_range
+    size = 15
+    return (size, aspect_ratio) if aspect_ratio > 1 else (size / aspect_ratio, size)
+
+
+def create_colorbar(fig, ax, dataset, coloring_col, cmap, title="", cb_positioning=[0.9, 0.4, 0.02, 0.38],
+                     tick_unit="", normalize_override=("min", "max")):
+    
+    divider = make_axes_locatable(ax)
+    divider.append_axes("right", size="2%", pad=5.55)
+
+    # Determine normalization values
+    if normalize_override[0] == "min":
+        vmin = dataset[coloring_col].min()
+    else:
+        vmin = normalize_override[0]
+
+    if normalize_override[1] == "max":
+        vmax = dataset[coloring_col].max()
+    else:
+        vmax = normalize_override[1]
+
+    sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=vmin, vmax=vmax))
+
+    colorbar_ax = fig.add_axes(cb_positioning)
+    colorbar = fig.colorbar(sm, cax=colorbar_ax)
+
+    min_tick = vmin
+    max_tick = vmax
+    colorbar.set_ticks([min_tick*1.05, max_tick*0.95])
+    colorbar.ax.set_yticklabels([
+                                 str(round(min_tick,1))+" " +tick_unit, 
+                                 str(round(max_tick,1)) + " " +tick_unit
+                                 ])
+    colorbar.ax.tick_params(labelsize=44)
+    
+
+    colorbar.ax.annotate(title , xy=(0.55, 1.1), xycoords='axes fraction', fontsize=44,
+                     xytext=(-45, 15), textcoords='offset points',
+                     ha='left', va='bottom')
+
+    for a in fig.axes:
+        if a is not ax and a is not colorbar_ax:
+            a.axis('off')
+
+    return sm, colorbar
+
+
+
+def draw_polygons(ax, dataset, x_cord_name, y_cord_name, style_dict, sm=None, drawing_order=None, cmap=None, coloring_col=None):
+    """
+    This function draws polygons on a given axes object based on coordinates defined in the dataset.
+
+    Parameters:
+    ax (matplotlib.axes.Axes): The axes object on which to draw the polygons.
+    dataset (pd.DataFrame): The input DataFrame containing the coordinates of the polygons.
+    x_cord_name (str): The name of the column in the dataset that contains the x-coordinates.
+    y_cord_name (str): The name of the column in the dataset that contains the y-coordinates.
+    style_dict (dict): A dictionary defining the style parameters for the polygons.
+    sm (matplotlib.cm.ScalarMappable, optional): The scalar mappable object used for mapping normalized data to RGBA.
+    drawing_order (list, optional): A list of indices defining the order in which to draw the polygons.
+    cmap (matplotlib.colors.Colormap, optional): The colormap to use for coloring the polygons.
+    coloring_col (str, optional): The name of the column in the dataset that contains the coloring values for the polygons.
+
+    Returns:
+    None
+
+    The function reads the x and y coordinates from the dataset and creates a polygon for each row. 
+    If a scalar mappable and a colormap are provided, the polygons are colored accordingly. 
+    The order in which the polygons are drawn can be specified with the drawing_order parameter. 
+    If no order is specified, the polygons are drawn in the order they appear in the dataset.
+    """
+    if drawing_order is None:
+        drawing_order = dataset.index
+    for idx in drawing_order:
+        row  = dataset.loc[idx]
+
+        # If it's a string, convert to list, if list, use directly
+        if isinstance(row[x_cord_name], str) and len(row[x_cord_name]) > 2:
+            patch_x_list = [float(i) for i in row[x_cord_name][1:-1].split(",")]
+        elif isinstance(row[x_cord_name], list):
+            patch_x_list = row[x_cord_name]
+
+        if isinstance(row[y_cord_name], str) and len(row[y_cord_name]) > 2:
+            patch_y_list = [float(i) for i in row[y_cord_name][1:-1].split(",")]
+        elif isinstance(row[y_cord_name], list):
+            patch_y_list = row[y_cord_name]
+
+        # Check if the row is not None and the length is greater than 0
+        if patch_x_list is not None and patch_y_list is not None and len(patch_x_list) > 0 and len(patch_y_list) > 0:
+            try:
+                if patch_x_list[0] != patch_x_list[-1] and patch_y_list[0] != patch_y_list[-1]:
+                    patch_x_list.append(patch_x_list[0])
+                    patch_y_list.append(patch_y_list[0])
+
+                if sm is not None:
+                    normalized_data = sm.norm(row[coloring_col])
+                    polygon = patches.Polygon(np.column_stack((patch_x_list, patch_y_list)), **style_dict, facecolor=cmap(normalized_data))
+
+                else:
+                    polygon = patches.Polygon(np.column_stack((patch_x_list, patch_y_list)), **style_dict)
+
+                ax.add_patch(polygon)
+            except Exception as e:
+               pass
+               #print(f"Error occurred: {e}")
+
+
+def configure_plot(ax, all_x_coords, all_y_coords, buffer=0.03):
+    x_range = max(all_x_coords) - min(all_x_coords)
+    y_range = max(all_y_coords) - min(all_y_coords)
+    
+    ax.set_aspect('equal')
+    ax.set_xlim([min(all_x_coords) - buffer*x_range, max(all_x_coords) + buffer*x_range])
+    ax.set_ylim([min(all_y_coords) - buffer*y_range, max(all_y_coords) + buffer*y_range])
+    ax.set_xticks([])
+    ax.set_yticks([])
+    for spine in ax.spines.values():
+        spine.set_visible(False)
+
+
+# Main script
+#dataset = dataset.dropna()
+
+# column used for heatmap and colorbar
+def createActivityNodePlot(dataset, 
+                           colorbar_title="", 
+                           color="coolwarm", 
+                           data_col=None, 
+                           cb_positioning = [0.9, 0.4, 0.02, 0.38], 
+                           draw_oder_instruction=['-', '-', '+'],
+                           tick_unit="",
+                           normalize_override=("min", "max")):
+    
+    """
+    This function creates an activity node plot using the provided dataset, and optionally includes a colorbar. 
+
+    Parameters:
+    dataset (pd.DataFrame): The input DataFrame containing the data.
+    colorbar_title (str, optional): The title for the colorbar. Default is an empty string.
+    color (str or list, optional): The colormap for the plot. Can be a matplotlib colormap name or a list of colors. Default is "coolwarm".
+    data_col (str, optional): The name of the column in the dataset to use for coloring the nodes. If not provided, the first column of the dataset is used.
+    cb_positioning (list, optional): A list of four floats defining the position and size of the colorbar. Defaults to [0.9, 0.4, 0.02, 0.38].
+    draw_oder_instruction (list, optional): A list of strings defining the order in which to draw the polygons. Defaults to ['-', '-', '+'].
+    tick_unit (str, optional): The unit for the ticks on the colorbar. Default is an empty string.
+
+    Returns:
+    fig (matplotlib.figure.Figure): The created figure object.
+    ax (matplotlib.axes._subplots.AxesSubplot): The created Axes object.
+
+    The function creates an activity node plot with optional coloring based on a data column. 
+    The plot includes polygons representing nodes, and optionally a colorbar. 
+    The order in which the nodes are drawn can be specified. 
+    The plot's aspect ratio is calculated based on the provided coordinates.
+    """
+    
+    if data_col == None:
+        coloring_col = dataset.columns[0]
+    else:
+        coloring_col = data_col
+
+    # not very elegant
+    all_x_coords = []
+    all_y_coords = []
+
+    for idx, row in dataset.iterrows():
+        # If it's a string, convert to list, if list, use directly
+        if isinstance(row["patches_x_AN"], str) and len(row["patches_x_AN"]) > 2:
+            patch_x_list = [float(i) for i in row["patches_x_AN"][1:-1].split(",")]
+        elif isinstance(row["patches_x_AN"], list):
+            patch_x_list = row["patches_x_AN"]
+
+        if isinstance(row["patches_y_AN"], str) and len(row["patches_y_AN"]) > 2:
+            patch_y_list = [float(i) for i in row["patches_y_AN"][1:-1].split(",")]
+        elif isinstance(row["patches_y_AN"], list):
+            patch_y_list = row["patches_y_AN"]
+        all_x_coords.extend(patch_x_list)
+        all_y_coords.extend(patch_y_list)
+    
+    figsize = calculate_aspect_ratio(all_x_coords, all_y_coords)
+    fig, ax = plt.subplots(figsize=figsize)
+
+    # color map
+    if type(color) == type([]):
+        
+        cmap = LinearSegmentedColormap.from_list('custom_color', color)
+    else:
+        cmap = plt.cm.get_cmap(color)
+
+    # Activity Node geometry
+    style_dict_an = {'linewidth': 1, 'edgecolor': "Black"} 
+
+    color_data_exists = is_numeric_dtype(dataset[coloring_col])
+
+    if color_data_exists:
+        sm, colorbar = create_colorbar(fig, ax, dataset, coloring_col, cmap, colorbar_title, 
+                                       cb_positioning = cb_positioning, tick_unit=tick_unit,
+                                       normalize_override=normalize_override)
+    drawing_order = get_drawing_order(dataset, [1, 3, 2], draw_oder_instruction)
+
+    draw_polygons(ax, 
+                dataset, 
+                "patches_x_AN", 
+                "patches_y_AN", 
+                style_dict_an, 
+                sm,
+                drawing_order,
+                cmap,
+                coloring_col)
+
+    style_dict_bridges = {'linewidth': 1, 'edgecolor': "Black", 'facecolor':"Black"} 
+
+
+    draw_polygons(ax, 
+                dataset, 
+                "patches_x_Bridges", 
+                "patches_y_Bridges", 
+                style_dict_bridges,
+                cmap,
+                coloring_col=coloring_col,
+                )
+
+    configure_plot(ax, all_x_coords, all_y_coords)
+    return fig, ax
+
+
+
+    
+def radar(values_norm,
+          labels,  
+          color, 
+          cluster_name, 
+          factor=100, 
+          ax_multi = None, 
+          fig_multi=None, 
+          label_font_size =6,
+          num_datapoints=None):
+        
+    """
+    This function creates a radar chart (also known as a spider or star chart) from given normalized values and labels.
+
+    Parameters:
+    values_norm (list of numbers): Normalized values to plot on the radar chart, these values will be scaled within the function.
+    labels (list of str): Labels for the axes of the radar chart.
+    color (str): Color of the fill and outline on the radar chart.
+    cluster_name (str): Title for the radar chart.
+    factor (int, optional): Scaling factor for the data, defaults to 100.
+    ax_multi (matplotlib Axes object, optional): Predefined matplotlib Axes. If None, a new Axes object is created.
+    fig_multi (matplotlib Figure object, optional): Predefined matplotlib Figure for the plot. If None, a new Figure is created.
+    label_font_size (int, optional): Font size for the axis labels, defaults to 6.
+    num_datapoints (int, optional): Number of datapoints used to calculate the values, will be displayed in the plot if provided.
+
+    Returns:
+    fig (matplotlib Figure object): Figure containing the radar chart.
+    ax (matplotlib Axes object): Axes of the created radar chart.
+
+    This function plots each value from 'values_norm' as an axis on the radar chart, 
+    the aesthetics of the plot such as color and font size are customizable. The chart 
+    is scaled using the provided factor. 'values_norm' should be preprocessed outside 
+    of this function: they should be the mean values of your original data, normalized 
+    to be between 0 and 1.
+    """
+
+    # ax = plt.subplot(polar=True)
+    if ax_multi == None or fig_multi == None:
+      fig, ax = plt.subplots(figsize=(3.5, 3.5), subplot_kw=dict(polar=True), dpi=200)
+    else:
+      fig = fig_multi
+      ax = ax_multi
+
+    values_norm = [v*factor for v in values_norm]
+
+    # Number of variables we're plotting.
+    num_vars = len(labels)
+
+    # Split the circle into even parts and save the angles
+    # so we know where to put each axis.
+    angles = np.linspace(0, 2 * np.pi, num_vars, endpoint=False).tolist()
+
+    # The plot is a circle, so we need to "complete the loop"
+    # and append the start value to the end.
+    values_norm += values_norm[:1]
+    angles += angles[:1]
+
+    # Draw the outline of our data.
+    ax.plot(angles, values_norm, color=color, linewidth=2)
+
+    # Fill it in.
+    ax.fill(angles, values_norm, color=color, alpha=0.15)
+
+    # Fix axis to go in the right order and start at 12 o'clock.
+    ax.set_theta_offset(np.pi / 2)
+    ax.set_theta_direction(-1)
+
+    # Draw axis lines for each angle and label.
+    labels += labels[:1]
+    ax.set_thetagrids(np.degrees(angles), labels)
+   
+    # Go through labels and adjust alignment based on where
+    # it is in the circle.
+    for label, angle in zip(ax.get_xticklabels(), angles):
+        if angle in (0, np.pi):
+            label.set_horizontalalignment('center')
+        elif 0 < angle < np.pi:
+            label.set_horizontalalignment('left')
+        else:
+            label.set_horizontalalignment('right')
+        label.set_fontsize(label_font_size)
+
+    # Ensure radar goes from 0 to 100.
+    ax.set_ylim(0, 100)
+
+    # of the first two axes.
+    ax.set_rlabel_position(180 / num_vars)
+
+    # Add some custom styling.
+    # Change the color of the tick labels.
+    ax.tick_params(colors='#222222')
+
+    # Make the y-axis (0-100) labels smaller.
+    ax.tick_params(axis='y', labelsize=6)
+    # Change the color of the circular gridlines.
+    ax.grid(color='#AAAAAA')
+    # Change the color of the outermost gridline (the spine).
+    ax.spines['polar'].set_color('#222222')
+    # Change the background color inside the circle itself.
+    ax.set_facecolor('#FAFAFA')
+
+    # Lastly, give the chart a title and give it some
+    # padding above the "Acceleration" label.
+    ax.set_title(cluster_name, y=1.11)
+
+     # Add this at the end of your function
+    if num_datapoints is not None:
+        # plt.figtext adds text to the figure as a whole, outside individual subplots
+        # The parameters are (x, y, text), where x and y are in figure coordinates
+        plt.figtext(0.5, -0.05, f'datapoints: {num_datapoints}', ha='center')
+
+    return fig, ax
+
+
+def gh_color_blueRed():
+    # grasshoper color scheme 
+    color_list = [[15,16,115],
+            [177,198,242],
+            [251,244,121],
+            [222,140,61],
+            [183,60,34]]
+    # Scale RGB values to [0,1] range
+    color_list = [[c/255. for c in color] for color in color_list]
+    return color_list
+
+
+def linear_regression_with_residuals(
+    df, x_name, y_name, buffer=5, data_range_max=None, max_residual_color=None, rescale_range=None, generateName=False
+    ):
+
+    """
+    Generate a scatter plot with linear regression, residuals, and a color-coded line of equality.
+
+    Parameters:
+    df (DataFrame): The DataFrame containing the data.
+    x_name (str): The name of the x-axis variable.
+    y_name (str): The name of the y-axis variable.
+    buffer (int, optional): Buffer as a percentage of data range for plot margins. Default is 5.
+    data_range_max (float, optional): Maximum value for x and y axes. Default is None (auto-calculated).
+    max_residual_color (float, optional): Maximum residual value for color normalization. Default is None (auto-calculated).
+    rescale_range (tuple, optional): Rescale both x and y to the specified range. Default is None (no rescaling).
+    save_png (str, optional): File path to save the plot as a PNG image. Default is None (no saving).
+    date_source (str, optional): Date source identifier for the filename. Default is None.
+
+    Returns:
+    plt: Matplotlib figure for the generated plot.
+    """
+
+    # Extract x and y values from the DataFrame
+    x = df[x_name].values
+    y = df[y_name].values
+
+    # Rescale x and y if rescale_range is provided
+    if rescale_range:
+        x_min, x_max = rescale_range
+        x = (x - min(x)) / (max(x) - min(x)) * (x_max - x_min) + x_min
+        y = (y - min(y)) / (max(y) - min(y)) * (x_max - x_min) + x_min
+
+    # Calculate R2 score
+    r2 = r2_score(x, y)
+    print(f"R2 Score: {r2}")
+
+    # Calculate residuals in relation to the 45-degree line
+    residuals_45 = y - x.flatten()
+
+    # Calculate the data range with a buffer
+    if data_range_max:
+        data_min = 0
+        data_max = data_range_max
+    else:
+        data_min = min(min(x), min(y))
+        data_max = max(max(x), max(y))
+    buffer_value = (data_max - data_min) * (buffer / 100)
+
+    # Create a square plot with the same range for both axes
+    plt.figure()
+    colormap = 'bwr'  # Choose a colormap
+    cmap = plt.get_cmap(colormap)
+    plt.rcParams['font.family'] = 'DejaVu Sans'
+
+    # Shift the midpoint of the colormap to zero
+    if max_residual_color is None:
+        max_residual_color = max(abs(residuals_45))
+    norm = plt.Normalize(-max_residual_color, max_residual_color)
+
+    colors = np.array(cmap(norm(residuals_45)), dtype=object)
+
+    # Darken the edge color by making it 90% darker than the fill color
+    edge_colors = [tuple(0.9 * np.array(c)) for c in colors]
+
+    # Add a contour to scatter points with the same color as the point fill
+    scatter = plt.scatter(x, y, c=colors, label='True values', edgecolors=edge_colors, linewidths=2, zorder=3)
+
+    # Plot the line of equality (x == y)
+    combined_line = plt.plot([data_min - buffer_value, data_max + buffer_value], [data_min - buffer_value, data_max + buffer_value],
+             color='black', linewidth=1, zorder=5)
+
+    # Calculate and plot residuals in relation to the line of equality
+    for i in range(len(x)):
+        plt.plot([x[i], x[i]], [y[i], x[i]], color='gray', linestyle='--', linewidth=0.5, zorder=1)
+
+    # Plot the linear regression line
+    m, b = np.polyfit(x, y, 1)
+    regression_line = plt.plot(x, m * x + b, color='grey', linestyle='dotted', linewidth=1, label='Linear Regression line', zorder=4)
+
+    # Calculate the R2 score text position
+    text_x = data_min + 0.01 * (data_max - data_min)
+    text_y = data_max - 0.01 * (data_max - data_min)
+
+    # Annotate the plot with the R2 score
+    plt.text(text_x, text_y, f'$R^2$ Score: {r2:.2f}', fontsize=8, color='black')
+
+    # Add colorbar for residuals (smaller and within the plot)
+    sm = plt.cm.ScalarMappable(cmap=colormap, norm=norm)
+    sm.set_array([])
+    cbar = plt.colorbar(sm, ax=plt.gca(), shrink=0.2, aspect=15, pad=0.03)
+    cbar.set_label('Residuals (line of Equality)', fontsize=8)
+
+    # Create separate legend handles and labels
+    legend_handles = [scatter, regression_line[0], combined_line[0]]
+    legend_labels = ['True values', 'Linear Regression line', 'Line of Equality']
+
+    # Create the combined legend
+    combined_legend = plt.legend(handles=legend_handles, labels=legend_labels, loc='lower right', fontsize=8)
+
+    # Set the same limits for both x and y axes with a buffer
+    plt.xlim(data_min - buffer_value, data_max + buffer_value)
+    plt.ylim(data_min - buffer_value, data_max + buffer_value)
+
+    plt.gca().add_artist(combined_legend)  # Add the combined legend to the plot
+
+    plt.title('Linear Regression Visualization with Residuals (line of Equality)')
+    plt.xlabel(" ".join(x_name.split("+"))[0].capitalize() + " ".join(x_name.split("+"))[1:])
+    plt.ylabel(" ".join(y_name.split("+"))[0].capitalize() + " ".join(y_name.split("+"))[1:])
+
+    # Add very light grey background grid lines
+    plt.grid(True, color='lightgrey', linestyle='--', alpha=0.6, zorder=0)
+
+
+    if generateName:
+        # Plot name
+        plt_name = "linearRegr_" + "".join(word.capitalize() for word in x_name.split("+")) + "_vs_" + "".join(
+            word.capitalize() for word in y_name.split("+"))
+        return plt, plt_name
+    else:
+        return plt

speckleUtils/speckle_utils.py

@@ -0,0 +1,696 @@
+#speckle utils
+import json 
+import pandas as pd
+import numpy as np
+import specklepy
+from specklepy.api.client import SpeckleClient
+from specklepy.api.credentials import get_default_account, get_local_accounts
+from specklepy.transports.server import ServerTransport
+from specklepy.api import operations
+from specklepy.objects.geometry import Polyline, Point, Mesh
+
+from specklepy.api.wrapper import StreamWrapper
+try:
+    import openai
+except:
+    pass
+
+import requests
+from datetime import datetime
+import copy
+
+
+# HELP FUNCTION ===============================================================
+def helper():
+    """
+    Prints out the help message for this module.
+    """
+    print("This module contains a set of utility functions for speckle streams.")
+    print("______________________________________________________________________")
+    print("It requires the specklepy package to be installed -> !pip install specklepy")
+    print("the following functions are available:")
+    print("getSpeckleStream(stream_id, branch_name, client)")
+    print("getSpeckleGlobals(stream_id, client)")
+    print("get_dataframe(objects_raw, return_original_df)")
+    print("updateStreamAnalysis(stream_id, new_data, branch_name, geometryGroupPath, match_by_id, openai_key, return_original)")
+    print("there are some more function available not documented fully yet, including updating a notion database")
+    print("______________________________________________________________________")
+    print("for detailed help call >>> help(speckle_utils.function_name) <<< ")
+    print("______________________________________________________________________")
+    print("standard usage:")
+    print("______________________________________________________________________")
+    print("retreiving data")
+    print("1. import speckle_utils & speckle related libaries from specklepy")
+    print("2. create a speckle client -> client = SpeckleClient(host='https://speckle.xyz/')" )
+    print("                              client.authenticate_with_token(token='your_token_here')")
+    print("3. get a speckle stream -> stream = speckle_utils.getSpeckleStream(stream_id, branch_name, client)")
+    print("4. get the stream data -> data = stream['pth']['to']['data']")
+    print("5. transform data to dataframe -> df = speckle_utils.get_dataframe(data, return_original_df=False)")
+    print("______________________________________________________________________")
+    print("updating data")
+    print("1. call updateStreamAnalysis --> updateStreamAnalysis(new_data, stream_id, branch_name, geometryGroupPath, match_by_id, openai_key, return_original)")
+
+
+#==============================================================================
+
+def updateSpeckleStream(stream_id,
+                        branch_name,
+                        client,
+                        data_object,
+                        commit_message="Updated the data object",
+                        ):
+    """
+    Updates a speckle stream with a new data object.
+
+    Args:
+        stream_id (str): The ID of the speckle stream.
+        branch_name (str): The name of the branch within the speckle stream.
+        client (specklepy.api.client.Client): A speckle client.
+        data_object (dict): The data object to send to the speckle stream.
+        commit_message (str): The commit message. Defaults to "Updated the data object".
+    """
+    # set stream and branch
+    branch = client.branch.get(stream_id, branch_name)
+    # Get transport
+    transport = ServerTransport(client=client, stream_id=stream_id)
+    # Send the data object to the speckle stream
+    object_id = operations.send(data_object, [transport])
+
+    # Create a new commit with the new object
+    commit_id = client.commit.create(
+        stream_id,
+        object_id= object_id,
+        message=commit_message,
+        branch_name=branch_name,
+    )
+
+    return commit_id
+def getSpeckleStream(stream_id,
+                     branch_name,
+                     client,
+                     commit_id=""
+                     ):
+    """
+    Retrieves data from a specific branch of a speckle stream.
+
+    Args:
+        stream_id (str): The ID of the speckle stream.
+        branch_name (str): The name of the branch within the speckle stream.
+        client (specklepy.api.client.Client, optional): A speckle client. Defaults to a global `client`.
+        commit_id (str): id of a commit, if nothing is specified, the latest commit will be fetched
+
+    Returns:
+        dict: The speckle stream data received from the specified branch.
+
+    This function retrieves the last commit from a specific branch of a speckle stream.
+    It uses the provided speckle client to get the branch and commit information, and then 
+    retrieves the speckle stream data associated with the last commit.
+    It prints out the branch details and the creation dates of the last three commits for debugging purposes.
+    """
+
+    print("updated A")
+
+    # set stream and branch
+    try:
+        branch = client.branch.get(stream_id, branch_name, 3)
+        print(branch)
+    except:
+        branch = client.branch.get(stream_id, branch_name, 1)
+        print(branch)
+
+    print("last three commits:")
+    [print(ite.createdAt) for ite in branch.commits.items]
+
+    if commit_id == "":
+        latest_commit = branch.commits.items[0]
+        choosen_commit_id = latest_commit.id
+        commit = client.commit.get(stream_id, choosen_commit_id)
+        print("latest commit ", branch.commits.items[0].createdAt, " was choosen")
+    elif type(commit_id) == type("s"): # string, commit uuid
+        choosen_commit_id = commit_id
+        commit = client.commit.get(stream_id, choosen_commit_id)
+        print("provided commit ", choosen_commit_id, " was choosen")
+    elif type(commit_id) == type(1): #int 
+        latest_commit = branch.commits.items[commit_id]
+        choosen_commit_id = latest_commit.id
+        commit = client.commit.get(stream_id, choosen_commit_id)
+
+
+    print(commit)
+    print(commit.referencedObject)
+    # get transport
+    transport = ServerTransport(client=client, stream_id=stream_id)
+    #speckle stream
+    res = operations.receive(commit.referencedObject, transport)
+
+    return res
+ 
+def getSpeckleGlobals(stream_id, client):
+    """
+    Retrieves global analysis information from the "globals" branch of a speckle stream.
+
+    Args:
+        stream_id (str): The ID of the speckle stream.
+        client (specklepy.api.client.Client, optional): A speckle client. Defaults to a global `client`.
+
+    Returns:
+        analysisInfo (dict or None): The analysis information retrieved from globals. None if no globals found.
+        analysisGroups (list or None): The analysis groups retrieved from globals. None if no globals found.
+
+    This function attempts to retrieve and parse the analysis information from the "globals" 
+    branch of the specified speckle stream. It accesses and parses the "analysisInfo" and "analysisGroups" 
+    global attributes, extracts analysis names and UUIDs.
+    If no globals are found in the speckle stream, it returns None for both analysisInfo and analysisGroups.
+    """
+    # get the latest commit
+    try:
+        # speckle stream globals
+        branchGlob = client.branch.get(stream_id, "globals")
+        latest_commit_Glob = branchGlob.commits.items[0]
+        transport = ServerTransport(client=client, stream_id=stream_id)
+
+        globs = operations.receive(latest_commit_Glob.referencedObject, transport)
+        
+        # access and parse globals
+        #analysisInfo = json.loads(globs["analysisInfo"]["@{0;0;0;0}"][0].replace("'", '"'))
+        #analysisGroups = [json.loads(gr.replace("'", '"')) for gr in globs["analysisGroups"]["@{0}"]]
+
+        def get_error_context(e, context=100):
+            start = max(0, e.pos - context)
+            end = e.pos + context
+            error_line = e.doc[start:end]
+            pointer_line = ' ' * (e.pos - start - 1) + '^'
+            return error_line, pointer_line
+
+        try:
+            analysisInfo = json.loads(globs["analysisInfo"]["@{0;0;0;0}"][0].replace("'", '"').replace("None", "null"))
+        except json.JSONDecodeError as e:
+            print(f"Error decoding analysisInfo: {e}")
+            error_line, pointer_line = get_error_context(e)
+            print("Error position and surrounding text:")
+            print(error_line)
+            print(pointer_line)
+            analysisInfo = None
+
+        try:
+            analysisGroups = [json.loads(gr.replace("'", '"').replace("None", "null")) for gr in globs["analysisGroups"]["@{0}"]]
+        except json.JSONDecodeError as e:
+            print(f"Error decoding analysisGroups: {e}")
+            error_line, pointer_line = get_error_context(e)
+            print("Error position and surrounding text:")
+            print(error_line)
+            print(pointer_line)
+            analysisGroups = None
+
+
+
+        # extract analysis names 
+        analysis_names = []
+        analysis_uuid = []
+        [(analysis_names.append(key.split("++")[0]),analysis_uuid.append(key.split("++")[1]) ) for key in analysisInfo.keys()]
+
+
+        # print extracted results
+        print("there are global dictionaries with additional information for each analysis")
+        print("<analysisGroups> -> ", [list(curgrp.keys()) for curgrp in analysisGroups])
+        print("<analysis_names> -> ", analysis_names)                       
+        print("<analysis_uuid>  -> ", analysis_uuid)
+    except Exception as e:  # catch exception as 'e'
+        analysisInfo = None
+        analysisGroups = None
+        print("No GlOBALS FOUND")
+        print(f"Error: {e}")  # print error description
+  
+    return analysisInfo, analysisGroups
+
+
+
+#function to extract non geometry data from speckle 
+def get_dataframe(objects_raw, return_original_df=False):
+    """
+    Creates a pandas DataFrame from a list of raw Speckle objects.
+
+    Args:
+        objects_raw (list): List of raw Speckle objects.
+        return_original_df (bool, optional): If True, the function also returns the original DataFrame before any conversion to numeric. Defaults to False.
+
+    Returns:
+        pd.DataFrame or tuple: If return_original_df is False, returns a DataFrame where all numeric columns have been converted to their respective types, 
+                               and non-numeric columns are left unchanged. 
+                               If return_original_df is True, returns a tuple where the first item is the converted DataFrame, 
+                               and the second item is the original DataFrame before conversion.
+
+    This function iterates over the raw Speckle objects, creating a dictionary for each object that excludes the '@Geometry' attribute. 
+    These dictionaries are then used to create a pandas DataFrame. 
+    The function attempts to convert each column to a numeric type if possible, and leaves it unchanged if not. 
+    Non-convertible values in numeric columns are replaced with their original values.
+    """
+    # dataFrame
+    df_data = []
+    # Iterate over speckle objects
+    for obj_raw in objects_raw:
+        obj = obj_raw.__dict__
+        df_obj = {k: v for k, v in obj.items() if k != '@Geometry'}
+        df_data.append(df_obj)
+
+    # Create DataFrame and GeoDataFrame
+    df = pd.DataFrame(df_data)
+    # Convert columns to float or int if possible, preserving non-convertible values <-
+    df_copy = df.copy()
+    for col in df.columns:
+        df[col] = pd.to_numeric(df[col], errors='coerce')
+        df[col].fillna(df_copy[col], inplace=True)
+
+    if return_original_df:
+        return df, df_copy
+    else:
+        return df
+    
+
+def updateStreamAnalysis(
+          client,
+          new_data,
+          stream_id,
+          branch_name,
+          geometryGroupPath=None,
+          match_by_id="",
+          openai_key ="",
+          return_original = False
+      ):
+  
+
+    """
+    Updates Stream Analysis by modifying object attributes based on new data.
+
+    Args:
+        new_data (pandas.DataFrame): DataFrame containing new data.
+        stream_id (str): Stream ID.
+        branch_name (str): Branch name.
+        geometry_group_path (list, optional): Path to geometry group. Defaults to ["@Data", "@{0}"].
+        match_by_id (str, optional): key for column that should be used for matching. If empty, the index is used.
+        openai_key (str, optional): OpenAI key. If empty no AI commit message is generated Defaults to an empty string.
+        return_original (bool, optional): Determines whether to return original speckle stream objects. Defaults to False.
+
+    Returns:
+        list:  original speckle stream objects as backup if return_original is set to True.
+
+    This function retrieves the latest commit from a specified branch, obtains the 
+    necessary geometry objects, and matches new data with existing objects using 
+    an ID mapper. The OpenAI GPT model is optionally used to create a commit summary 
+    message. Changes are sent back to the server and a new commit is created, with 
+    the original objects returned as a backup if return_original is set to True. 
+    The script requires active server connection, necessary permissions, and relies 
+    on Speckle and OpenAI's GPT model libraries.
+    """
+    print("1")
+    if geometryGroupPath == None:
+        geometryGroupPath = ["@Speckle", "Geometry"]
+
+    branch = client.branch.get(stream_id, branch_name, 2)
+
+    latest_commit = branch.commits.items[0]
+    commitID = latest_commit.id 
+
+    commit = client.commit.get(stream_id, commitID)
+
+    # get objects
+    transport = ServerTransport(client=client, stream_id=stream_id)
+
+    #speckle stream
+    res = operations.receive(commit.referencedObject, transport)
+
+    # get geometry objects (they carry the attributes)
+    objects_raw = res[geometryGroupPath[0]][geometryGroupPath[1]]
+    res_new = copy.deepcopy(res)
+    print("2")
+    # map ids 
+    id_mapper = {}
+    if match_by_id != "":
+        for i, obj in enumerate(objects_raw):
+            id_mapper[obj[match_by_id]] = i
+    else:
+        for i, obj in enumerate(objects_raw):
+            id_mapper[str(i)] = i
+    print("3")
+    # iterate through rows (objects)
+    for index, row in new_data.iterrows():
+        #determin target object 
+        if match_by_id != "":
+            local_id = row[match_by_id]
+        else:
+            local_id = index
+        target_id = id_mapper[local_id]     
+
+        #iterate through columns (attributes)
+        for col_name in new_data.columns:
+            res_new[geometryGroupPath[0]][geometryGroupPath[1]][target_id][col_name] = row[col_name]
+
+    print("4")
+    # ======================== OPEN AI FUN ===========================
+    """
+    try:
+        try:
+            answer_summary = gptCommitMessage(objects_raw, new_data,openai_key)
+            if answer_summary == None:
+                _, answer_summary = compareStats(get_dataframe(objects_raw),new_data)
+        except:
+            _, answer_summary = compareStats(get_dataframe(objects_raw),new_data)
+    except:
+        answer_summary = ""
+    """
+    answer_summary = ""
+    # ================================================================
+    print("5")
+    new_objects_raw_speckle_id = operations.send(base=res_new, transports=[transport])
+    print("6")
+    # You can now create a commit on your stream with this object
+    commit_id = client.commit.create(
+        stream_id=stream_id,
+        branch_name=branch_name,
+        object_id=new_objects_raw_speckle_id,
+        message="Updated item in colab -" + answer_summary,
+        )
+    print("7")
+    print("Commit created!")
+    if return_original:
+        return objects_raw #as back-up
+
+def custom_describe(df):
+    # Convert columns to numeric if possible
+    df = df.apply(lambda x: pd.to_numeric(x, errors='ignore'))
+
+    # Initial describe with 'include = all'
+    desc = df.describe(include='all')
+
+    # Desired statistics
+    desired_stats = ['count', 'unique', 'mean', 'min', 'max']
+
+    # Filter for desired statistics
+    result = desc.loc[desired_stats, :].copy()
+    return result
+
+def compareStats(df_before, df_after):
+  """
+    Compares the descriptive statistics of two pandas DataFrames before and after some operations.
+
+    Args:
+        df_before (pd.DataFrame): DataFrame representing the state of data before operations.
+        df_after (pd.DataFrame): DataFrame representing the state of data after operations.
+
+    Returns:
+        The CSV string includes column name, intervention type, and before and after statistics for each column.
+        The summary string provides a count of updated and new columns.
+
+    This function compares the descriptive statistics of two DataFrames: 'df_before' and 'df_after'. 
+    It checks the columns in both DataFrames and categorizes them as either 'updated' or 'new'.
+    The 'updated' columns exist in both DataFrames while the 'new' columns exist only in 'df_after'.
+    For 'updated' columns, it compares the statistics before and after and notes the differences.
+    For 'new' columns, it lists the 'after' statistics and marks the 'before' statistics as 'NA'.
+    The function provides a summary with the number of updated and new columns, 
+    and a detailed account in CSV format of changes in column statistics.
+  """
+   
+  desc_before = custom_describe(df_before)
+  desc_after = custom_describe(df_after)
+
+  # Get union of all columns
+  all_columns = set(desc_before.columns).union(set(desc_after.columns))
+
+  # Track number of updated and new columns
+  updated_cols = 0
+  new_cols = 0
+
+  # Prepare DataFrame output
+  output_data = []
+
+  for column in all_columns:
+      row_data = {'column': column}
+      stat_diff = False  # Track if there's a difference in stats for a column
+
+      # Check if column exists in both dataframes
+      if column in desc_before.columns and column in desc_after.columns:
+          updated_cols += 1
+          row_data['interventionType'] = 'updated'
+          for stat in desc_before.index:
+              before_val = round(desc_before.loc[stat, column], 1) if pd.api.types.is_number(desc_before.loc[stat, column]) else desc_before.loc[stat, column]
+              after_val = round(desc_after.loc[stat, column], 1) if pd.api.types.is_number(desc_after.loc[stat, column]) else desc_after.loc[stat, column]
+              if before_val != after_val:
+                  stat_diff = True
+                  row_data[stat+'_before'] = before_val
+                  row_data[stat+'_after'] = after_val
+      elif column in desc_after.columns:
+          new_cols += 1
+          stat_diff = True
+          row_data['interventionType'] = 'new'
+          for stat in desc_after.index:
+              row_data[stat+'_before'] = 'NA'
+              after_val = round(desc_after.loc[stat, column], 1) if pd.api.types.is_number(desc_after.loc[stat, column]) else desc_after.loc[stat, column]
+              row_data[stat+'_after'] = after_val
+
+      # Only add to output_data if there's actually a difference in the descriptive stats between "before" and "after".
+      if stat_diff:
+          output_data.append(row_data)
+
+  output_df = pd.DataFrame(output_data)
+  csv_output = output_df.to_csv(index=False)
+  print (output_df)
+  # Add summary to beginning of output
+  summary = f"Summary:\n  Number of updated columns: {updated_cols}\n  Number of new columns: {new_cols}\n\n"
+  csv_output = summary + csv_output
+
+  return csv_output, summary
+
+
+
+# Function to call ChatGPT API
+def ask_chatgpt(prompt, model="gpt-3.5-turbo", max_tokens=300, n=1, stop=None, temperature=0.3):
+    import openai
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": "You are a helpfull assistant,."},
+            {"role": "user", "content": prompt}
+        ],
+        max_tokens=max_tokens,
+        n=n,
+        stop=stop,
+        temperature=temperature,
+    )
+    return response.choices[0].message['content']
+
+
+
+
+def gptCommitMessage(objects_raw, new_data,openai_key):
+    # the idea is to automatically create commit messages. Commits coming through this channel are all
+    # about updating or adding a dataTable. So we can compare the descriptive stats of a before and after
+    # data frame 
+    #try:
+    try:
+        import openai
+        openai.api_key = openai_key
+    except NameError as ne:
+        if str(ne) == "name 'openai' is not defined":
+            print("No auto commit message: openai module not imported. Please import the module before setting the API key.")
+        elif str(ne) == "name 'openai_key' is not defined":
+            print("No auto commit message: openai_key is not defined. Please define the variable before setting the API key.")
+        else:
+            raise ne
+
+    report, summary = compareStats(get_dataframe(objects_raw),new_data)
+
+    # prompt
+    prompt = f"""Given the following changes in my tabular data structure, generate a 
+    precise and informative commit message. The changes involve updating or adding 
+    attribute keys and values. The provided summary statistics detail the changes in 
+    the data from 'before' to 'after'. 
+    The CSV format below demonstrates the structure of the summary:
+
+    Summary:
+    Number of updated columns: 2
+    Number of new columns: 1
+    column,interventionType,count_before,count_after,unique_before,unique_after,mean_before,mean_after,min_before,min_after,max_before,max_after
+    A,updated,800,800,2,3,,nan,nan,nan,nan,nan
+    B,updated,800,800,3,3,,nan,nan,nan,nan,nan
+    C,new,NA,800,NA,4,NA,nan,NA,nan,NA,nan
+
+    For the commit message, your focus should be on changes in the data structure, not the interpretation of the content. Be precise, state the facts, and highlight significant differences or trends in the statistics, such as shifts in mean values or an increase in unique entries.
+
+    Based on the above guidance, draft a commit message using the following actual summary statistics:
+
+    {report}
+
+    Your commit message should follow this structure:
+
+    1. Brief description of the overall changes.
+    2. Significant changes in summary statistics (count, unique, mean, min, max).
+    3. Conclusion, summarizing the most important findings with the strucutre:
+    # changed columns: , comment: ,
+    # added Columns:  , comment: ,
+    # Chaged statistic: ,  coment: ,
+
+    Mark the beginning of the conclusion with ">>>" and ensure to emphasize hard facts and significant findings. 
+    """
+
+    try:
+        answer = ask_chatgpt(prompt)
+        answer_summery = answer.split(">>>")[1]
+        if answer == None:
+            answer_summery = summary
+    except:
+        answer_summery = summary
+
+    print(answer_summery)
+    return answer_summery
+
+def specklePolyline_to_BokehPatches(speckle_objs, pth_to_geo="curves", id_key="ids"):
+  """
+  Takes a list of speckle objects, extracts the polyline geometry at the specified path, and returns a dataframe of x and y coordinates for each polyline.
+  This format is compatible with the Bokeh Patches object for plotting.
+  
+  Args:
+    speckle_objs (list): A list of Speckle Objects
+    pth_to_geo (str): Path to the geometry in the Speckle Object
+    id_key (str): The key to use for the uuid in the dataframe. Defaults to "uuid"
+    
+  Returns:
+    pd.DataFrame: A Pandas DataFrame with columns "uuid", "patches_x" and "patches_y"
+  """
+  patchesDict = {"uuid":[], "patches_x":[], "patches_y":[]}
+  
+  for obj in speckle_objs:
+    obj_geo = obj[pth_to_geo]
+    obj_pts = Polyline.as_points(obj_geo)
+    coorX = []
+    coorY = []
+    for pt in obj_pts:
+      coorX.append(pt.x)
+      coorY.append(pt.y)
+    
+    patchesDict["patches_x"].append(coorX)
+    patchesDict["patches_y"].append(coorY)
+    patchesDict["uuid"].append(obj[id_key])
+
+  return pd.DataFrame(patchesDict)
+
+
+
+def rebuildAnalysisInfoDict(analysisInfo):
+    """rebuild the analysisInfo dictionary to remove the ++ from the keys
+
+    Args:
+        analysisInfo (list): a list containing the analysisInfo dictionary
+
+    Returns:
+        dict: a dictionary containing the analysisInfo dictionary with keys without the ++
+
+    """
+    analysisInfoDict = {}
+    for curKey in analysisInfo[0]:
+        newkey = curKey.split("++")[0]
+        analysisInfoDict[newkey] = analysisInfo[0][curKey]
+    return analysisInfoDict
+
+
+def specklePolyline2Patches(speckle_objs, pth_to_geo="curves", id_key=None):
+    """
+    Converts Speckle objects' polyline information into a format suitable for Bokeh patches.
+
+    Args:
+        speckle_objs (list): A list of Speckle objects.
+        pth_to_geo (str, optional): The path to the polyline geometric information in the Speckle objects. Defaults to "curves".
+        id_key (str, optional): The key for object identification. Defaults to "uuid".
+
+    Returns:
+        DataFrame: A pandas DataFrame with three columns - "uuid", "patches_x", and "patches_y". Each row corresponds to a Speckle object.
+                    "uuid" column contains the object's identifier.
+                    "patches_x" and "patches_y" columns contain lists of x and y coordinates of the polyline points respectively.
+
+    This function iterates over the given Speckle objects, retrieves the polyline geometric information and the object's id from each Speckle object, 
+    and formats this information into a format suitable for Bokeh or matplotlib patches. The formatted information is stored in a dictionary with three lists 
+    corresponding to the "uuid", "patches_x", and "patches_y", and this dictionary is then converted into a pandas DataFrame.
+    """
+    patchesDict = {"patches_x":[], "patches_y":[]}
+    if id_key != None:
+        patchesDict[id_key] = []
+
+    for obj in speckle_objs:
+        obj_geo = obj[pth_to_geo]
+        
+        coorX = []
+        coorY = []
+        
+        if isinstance(obj_geo, Mesh):
+            # For meshes, we'll just use the vertices for now
+            for pt in obj_geo.vertices:
+                coorX.append(pt.x)
+                coorY.append(pt.y)
+        else:
+            # For polylines, we'll use the existing logic
+            obj_pts = Polyline.as_points(obj_geo)
+            for pt in obj_pts:
+                coorX.append(pt.x)
+                coorY.append(pt.y)
+
+        patchesDict["patches_x"].append(coorX)
+        patchesDict["patches_y"].append(coorY)
+        if id_key != None:
+            patchesDict[id_key].append(obj[id_key])
+
+    return pd.DataFrame(patchesDict)
+
+
+#================= NOTION INTEGRATION ============================
+headers = {
+    "Notion-Version": "2022-06-28",
+    "Content-Type": "application/json"
+}
+
+def get_page_id(token, database_id, name):
+    headers['Authorization'] = "Bearer " + token
+    # Send a POST request to the Notion API
+    response = requests.post(f"https://api.notion.com/v1/databases/{database_id}/query", headers=headers)
+
+    # Load the response data
+    data = json.loads(response.text)
+
+    # Check each page in the results
+    for page in data['results']:
+        # If the name matches, return the ID
+        if page['properties']['name']['title'][0]['text']['content'] == name:
+            return page['id']
+
+    # If no match was found, return None
+    return None
+
+def add_or_update_page(token, database_id, name, type, time_updated, comment, speckle_link):
+    # Format time_updated as a string 'YYYY-MM-DD'
+    date_string = time_updated.strftime('%Y-%m-%d')
+
+    # Construct the data payload
+    data = {
+        'parent': {'database_id': database_id},
+        'properties': {
+            'name': {'title': [{'text': {'content': name}}]},
+            'type': {'rich_text': [{'text': {'content': type}}]},
+            'time_updated': {'date': {'start': date_string}},
+            'comment': {'rich_text': [{'text': {'content': comment}}]},
+            'speckle_link': {'rich_text': [{'text': {'content': speckle_link}}]}
+        }
+    }
+
+    # Check if a page with this name already exists
+    page_id = get_page_id(token, database_id, name)
+
+    headers['Authorization'] = "Bearer " + token
+    if page_id:
+        # If the page exists, send a PATCH request to update it
+        response = requests.patch(f"https://api.notion.com/v1/pages/{page_id}", headers=headers, data=json.dumps(data))
+    else:
+        # If the page doesn't exist, send a POST request to create it
+        response = requests.post("https://api.notion.com/v1/pages", headers=headers, data=json.dumps(data))
+    
+    print(response.text)
+
+# Use the function
+#add_or_update_page('your_token', 'your_database_id', 'New Title', 'New Type', datetime.now(), 'This is a comment', 'https://your-link.com')

tripGenerationFunc.py

@@ -18,13 +18,16 @@ from functools import wraps
 
 
 
-sys.path.append("RECODE_speckle_utils")
+sys.path.append("speckleUtils")
 
 
-from .RECODE_speckle_utils import speckle_utils
-
+from .speckleUtils import speckle_utils
 
 
+#https://serjd-syncspeckle2notion.hf.space/webhooks/update_streams
+#https://serjd-RECODE_HF_tripGeneration.hf.space/webhooks/update_streams
+#serJD/RECODE_HF_tripGeneration
+# https://huggingface.co/spaces/serJD/RECODE_HF_tripGeneration
 
 # !!! lots of hard coded values in computeTrips !!!