Upload 4 files

app.py

@@ -0,0 +1,204 @@
+import json
+import re
+import sys
+import time
+import copy
+from notion_client import Client
+
+from specklepy.api.client import SpeckleClient
+from specklepy.api.credentials import get_default_account, get_local_accounts
+from specklepy.transports.server import ServerTransport
+from specklepy.api import operations
+from specklepy.objects.geometry import Polyline, Point
+from specklepy.objects import Base
+import os
+from functools import wraps
+
+import gradio as gr
+import requests
+from huggingface_hub import webhook_endpoint, WebhookPayload
+from fastapi import Request
+import datetime
+
+from huggingface_hub import WebhooksServer, WebhookPayload
+import gradio as gr
+
+
+from utils import *
+
+current_directory = os.path.dirname(os.path.abspath(__file__))
+# Path to the config.json file
+config_file_path = os.path.join(current_directory, "config.json")
+with open(config_file_path, 'r') as f:
+        config = json.load(f)
+
+
+
+
+
+# notion login
+speckle_token = os.environ.get("SPECKLE_TOKEN")
+notion_token = os.environ.get("NOTION_TOKEN")
+webhook_secret = os.environ.get("WEBHOOK_SECRET")
+notion_token = notion_token
+notion = Client(auth=notion_token)
+
+# speckle 
+CLIENT = SpeckleClient(host="https://speckle.xyz/")
+CLIENT.authenticate_with_token(token=speckle_token)
+
+# load config to variables 
+
+
+
+def mainFunc(STREAM_ID, SOURCE_BRANCH, TARGET_BRANCH, UUID_COL, ATTR_METADATA, KPI_METADATA, DEFAULT_ATTRIBUTES):
+
+    if type(TARGET_BRANCH) == type([]):
+        TARGET_BRANCH = SOURCE_BRANCH.replace(TARGET_BRANCH[0], TARGET_BRANCH[1])
+    
+    # get data from notion
+    databaseFUll_pages  = fetch_all_database_pages(notion, ATTR_METADATA)
+    kpi_database_pages = fetch_all_database_pages(notion, KPI_METADATA)
+
+    # generate JSON files
+    attributeMetaData, availableAttributes = notionTable2JSON(databaseFUll_pages, kpi_database_pages)
+
+    # extract attribute/colum/feature names from notion table 
+    attributesOfInterest = DEFAULT_ATTRIBUTES
+    for page in databaseFUll_pages:
+        pv = get_property_value(page, "name")
+        attributesOfInterest.append(pv)
+        if UUID_COL not in attributesOfInterest:
+            attributesOfInterest.append(UUID_COL)
+
+
+    # get speckle data 
+            # get stream
+    stream = getSpeckleStream(STREAM_ID,
+                            SOURCE_BRANCH,
+                            CLIENT,
+                            commit_id = "")
+
+    # navigate to list with speckle objects of interest
+    try:
+      stream_data = stream["@Data"]["@{0}"]
+    except:
+       print("something went wrong, try again with non-capital d")
+    try:
+     stream_data = stream["@data"]["@{0}"]
+    except:
+        print("check on speckle.com how to access the data")
+    
+
+    # ======== assamble new stream data ============
+
+    streamData_new = []
+    log = {"removedDatapoints":0,"removedID":[], "avg_attrRemoved":0, "removedAttr":[]}
+    for i, obj in enumerate(stream_data):
+        objDict = obj.__dict__
+        # REMOVE DATA POINTS ==============================
+        # COND A: no landuses at all.
+        pass_flag = True
+
+        tot_sqm =0
+        for k, v in objDict.items():
+            if k.startswith("lu+"):
+                tot_sqm += float(v)
+        if tot_sqm <= 0:
+            pass_flag = False 
+        
+
+        if pass_flag == False :
+            log["removedDatapoints"] +=1
+            log["removedID"].append(objDict[UUID_COL])
+            continue
+
+        # REMOVE ATTRIBUTES ===============================
+        datanew = Base()
+        for k, v in objDict.items():
+            if k in attributesOfInterest:
+                datanew[k] = v
+            else:
+                log["avg_attrRemoved"] +=1
+                log["removedAttr"].append(k)
+        
+
+        streamData_new.append(datanew)
+
+        log["avg_attrRemoved"] = log["avg_attrRemoved"]/len(stream_data)
+        log["removedAttr"] = list(set(log["removedAttr"]))
+
+        stream_new = copy.deepcopy(stream)
+
+        stream_new["@Data"]["@{0}"] = streamData_new
+        #add additional data to stream
+        stream_new["logs"] = json.dumps(log)
+        stream_new["attributeMetaData"] = json.dumps(attributeMetaData)
+        stream_new["availableAttributes"] = json.dumps(availableAttributes)
+
+
+        # set stream and branch
+        # Get transport
+        transport = ServerTransport(client=CLIENT, stream_id=STREAM_ID)
+        # Send the data object to the speckle stream
+        object_id = operations.send(stream_new, [transport])
+
+        # Create a new commit with the new object
+        commit_id = CLIENT.commit.create(
+            STREAM_ID,
+            object_id= object_id,
+            message="auto commit, removed datapoints: " + str( log["removedDatapoints"]) + "avg. removed attributes: "+ str(log["avg_attrRemoved"]),
+            branch_name=TARGET_BRANCH,
+        )
+
+        print(commit_id)
+
+
+
+@webhook_endpoint
+async def update_streams(request: Request):
+    # Initialize flag
+    should_continue = False
+
+    # Read the request body as JSON
+    payload = await request.json()
+
+    print("============= payload =============")
+    print(payload)
+    print("============= payload =============")
+
+    payload = payload["payload"]
+    # Check if the payload structure matches the expected format
+    if "event" in payload and "data" in payload["event"]:
+        event_data = payload["event"]["data"]
+
+        # Check if the event type is one of the specified types
+        if "event_name" in payload["event"] and payload["event"]["event_name"] in ["commit_create", "commit_delete", "commit_update"]:
+            # Check if the stream name matches the specified list
+            if "stream" in payload and payload["stream"]["name"] in config.keys():
+                streamName = payload["stream"]["name"]
+                # Check if the branch name matches the specified list
+                if "commit" in event_data and "branchName" in event_data["commit"]:
+                    branchName = event_data["commit"]["branchName"]
+                    # Attempt to load the configuration for the stream and branch
+                    try:
+                        stream_config = config[streamName][branchName]
+                        should_continue = True
+                        # You can now use stream_config for further processing
+                    except KeyError:
+                        print(f"Configuration for stream {streamName} and branch {branchName} not found.")
+                else:
+                    print("Branch name not found in payload.")
+            else:
+                print("Stream name not found or not in the specified list.")
+        else:
+            print("Event type is not one of the specified types.")
+    else:
+        print("Payload structure does not match the expected format.")
+
+    # If the flag is True, continue running the main part of the code
+    if should_continue:
+        locals().update(config[streamName, branchName])
+
+        # Your main code logic goes here
+        mainFunc(STREAM_ID, SOURCE_BRANCH, TARGET_BRANCH, UUID_COL, ATTR_METADATA, KPI_METADATA, DEFAULT_ATTRIBUTES)

config.json

@@ -0,0 +1,15 @@
+{
+    "activeSetting": "2BU_100",
+    "2BU_100":
+    {   
+        "STREAM_ID":"ebcfc50abe",
+        "SOURCE_BRANCH": "analysis_results/default+activity_node_analysis",
+        "TARGET_BRANCH": ["analysis_results", "dashboard"],
+        "UUID_COL": "ids",
+        "ATTR_METADATA": "ec2a636f079d4d7686f94901b6238242",
+        "KPI_METADATA": "c5e00b7b51504e5a9326befb9c54671c",
+        "DEFAULT_ATTRIBUTES":["@geometry","@Geometry"]
+    }
+
+
+}

requirements.txt

@@ -0,0 +1,2 @@
+specklepy
+notion-client

utils.py

@@ -0,0 +1,264 @@
+
+import json
+import re
+from specklepy.transports.server import ServerTransport
+from specklepy.api import operations
+
+def get_database_properties(database_pages):
+    # Iterate through the results (each page corresponds to a row in the database)
+    propList = []
+    for page in database_pages:
+         # Print the name and type of each property in this page
+        for prop_name, prop_data in page['properties'].items():
+            prop_type = prop_data['type']
+            propList.append(prop_name)
+        break
+    return propList
+
+# query full database
+def fetch_all_database_pages(client, database_id):
+    """
+    Fetches all pages from a specified Notion database.
+
+    :param client: Initialized Notion client.
+    :param database_id: The ID of the Notion database to query.
+    :return: A list containing all pages from the database.
+    """
+    start_cursor = None
+    all_pages = []
+
+    while True:
+        response = client.databases.query(
+            **{
+                "database_id": database_id,
+                "start_cursor": start_cursor
+            }
+        )
+
+        all_pages.extend(response['results'])
+
+        # Check if there's more data to fetch
+        if response['has_more']:
+            start_cursor = response['next_cursor']
+        else:
+            break
+
+    return all_pages
+
+
+def get_property_value(page, property_name):
+    """
+    Extracts the value from a specific property in a Notion page based on its type.
+    :param page: The Notion page data as retrieved from the API.
+    :param property_name: The name of the property whose value is to be fetched.
+    :return: The value or values contained in the specified property, depending on type.
+    """
+    # Check if the property exists in the page
+    if property_name not in page['properties']:
+        return None  # or raise an error if you prefer
+
+    property_data = page['properties'][property_name]
+    prop_type = property_data['type']
+
+    # Handle 'title' and 'rich_text' types
+    if prop_type in ['title', 'rich_text']:
+        return ''.join(text_block['text']['content'] for text_block in property_data[prop_type])
+
+    # Handle 'number' type
+    elif prop_type == 'number':
+        return property_data[prop_type]
+
+    # Handle 'select' type
+    elif prop_type == 'select':
+        return property_data[prop_type]['name'] if property_data[prop_type] else None
+
+    # Handle 'multi_select' type
+    elif prop_type == 'multi_select':
+        return [option['name'] for option in property_data[prop_type]]
+
+    # Handle 'date' type
+    elif prop_type == 'date':
+        if property_data[prop_type]['end']:
+            return (property_data[prop_type]['start'], property_data[prop_type]['end'])
+        else:
+            return property_data[prop_type]['start']
+
+    # Handle 'relation' type
+    elif prop_type == 'relation':
+        return [relation['id'] for relation in property_data[prop_type]]
+
+    # Handle 'people' type
+    elif prop_type == 'people':
+        return [person['name'] for person in property_data[prop_type] if 'name' in person]
+
+    # Add more handlers as needed for other property types
+
+    else:
+        # Return None or raise an error for unsupported property types
+        return None
+
+
+def parse_invalid_json(json_string):
+    if json_string == None or json_string == "none":
+      return None
+    # Replace fancy quotes and single quotes with standard double quotes
+    json_string = re.sub(r"[‘’“”]", '"', json_string)
+    json_string = json_string.replace("'", '"')
+
+    # Add quotes around any unquoted keys
+    json_string = re.sub(r'(?<!")(\b\w+\b)(?!"):', r'"\1":', json_string)
+
+    # Handle unquoted numeric values or booleans if necessary
+    # This part can be customized based on specific requirements
+
+    try:
+        # Try parsing the corrected string
+        return json.loads(json_string)
+    except json.JSONDecodeError as e:
+        # Handle parsing error (or re-raise the exception)
+        print("JSON parsing error:", e)
+        print(json_string)
+        return None
+
+def notionTable2JSON(databaseFUll_pages, kpi_database_pages):
+  attributeMetaData = {}
+  availableAttributes = {}
+  cnt = 0
+  for page in databaseFUll_pages:
+    attributeData = {
+        "name": get_property_value(page, "name"),
+        "nameShort": get_property_value(page, "nameShort"),
+        "nameLong": get_property_value(page, "nameLong"),
+        "description": get_property_value(page, "description"),
+        "indicator": get_property_value(page, "indicator"),
+        "unit": get_property_value(page, "unit"),
+        "unitShort": get_property_value(page, "unitShort"),
+        "spatialUnit": get_property_value(page, "spatialUnit"),
+        "method": get_property_value(page, "method"),
+        "type": get_property_value(page, "type"),
+        "colorMapping": parse_invalid_json(get_property_value(page, "colorMapping")),
+        "parameter": parse_invalid_json(get_property_value(page, "parameter")),
+        "level_1": get_property_value(page, "level_1"),
+        "level_2": get_property_value(page, "level_2"),
+        "level_3": get_property_value(page, "level_3"),
+
+        "dataSet": "ActivityNodes",
+        "dataSource": "",
+
+        "KPI": [],
+        "visualisation": []
+      }
+    curAttrName = get_property_value(page, "name")
+    print(curAttrName)
+    lev1 = get_property_value(page, "level_1")
+    lev2 = get_property_value(page, "level_2")
+    lev3= get_property_value(page, "level_3")
+
+    if lev1 != None and lev1 != "NA":
+        if lev1 not in availableAttributes:
+            availableAttributes[lev1] = {"sub-levels": {}, "values": []}
+
+        if lev2 != None and lev2 != "NA":
+            if lev2 not in availableAttributes[lev1]["sub-levels"]:
+                availableAttributes[lev1]["sub-levels"][lev2] = {"sub-levels": {}, "values": []}
+
+            if lev3 != None and lev3 != "NA":
+                if lev3 not in availableAttributes[lev1]["sub-levels"][lev2]["sub-levels"]:
+                    availableAttributes[lev1]["sub-levels"][lev2]["sub-levels"][lev3] = {"values": []}
+                availableAttributes[lev1]["sub-levels"][lev2]["sub-levels"][lev3]["values"].append(curAttrName)
+            else:
+                availableAttributes[lev1]["sub-levels"][lev2]["values"].append(curAttrName)
+        else:
+            availableAttributes[lev1]["values"].append(curAttrName)
+
+    # iterated through list of KPI ref. Ids
+    kpiIDs = get_property_value(page, "KPI")
+    for kpiID in kpiIDs:
+
+      curKPI = get_page_by_id(kpi_database_pages, kpiID)
+      KPI_template ={
+          "name":get_property_value(curKPI, "name"),
+          "type":get_property_value(curKPI, "type"),
+          "unit":get_property_value(curKPI, "unit"),
+          "color":parse_invalid_json(get_property_value(curKPI, "color")),
+          "nameShort": get_property_value(curKPI, "nameShort"),
+          "quality":get_property_value(curKPI, "quality"),
+          "args": parse_invalid_json(get_property_value(curKPI, "args")),
+          "description":get_property_value(curKPI, "description"),
+          "interpretrationHigh":get_property_value(curKPI, "interpretrationHigh"),
+          "interpretationLow":get_property_value(curKPI, "interpretationLow"),
+          }
+      # add KPI data to attributeData
+      attributeData["KPI"].append(KPI_template)
+
+    # add to main dictioanry
+    attributeMetaData[get_property_value(page, "name")] = attributeData
+
+  print("processed pages:", cnt)
+  return attributeMetaData, availableAttributes
+
+
+def get_page_by_id(notion_db_pages, page_id):
+  for pg in notion_db_pages:
+    if pg["id"] == page_id:
+      return pg
+
+def getSpeckleStream(stream_id,
+                     branch_name,
+                     client,
+                     commit_id=""
+                     ):
+    """
+    Retrieves data from a specific branch of a speckle stream.
+
+    Args:
+        stream_id (str): The ID of the speckle stream.
+        branch_name (str): The name of the branch within the speckle stream.
+        client (specklepy.api.client.Client, optional): A speckle client. Defaults to a global `client`.
+        commit_id (str): id of a commit, if nothing is specified, the latest commit will be fetched
+
+    Returns:
+        dict: The speckle stream data received from the specified branch.
+
+    This function retrieves the last commit from a specific branch of a speckle stream.
+    It uses the provided speckle client to get the branch and commit information, and then 
+    retrieves the speckle stream data associated with the last commit.
+    It prints out the branch details and the creation dates of the last three commits for debugging purposes.
+    """
+
+    print("updated A")
+
+    # set stream and branch
+    try:
+        branch = client.branch.get(stream_id, branch_name, 3)
+        print(branch)
+    except:
+        branch = client.branch.get(stream_id, branch_name, 1)
+        print(branch)
+
+    print("last three commits:")
+    [print(ite.createdAt) for ite in branch.commits.items]
+
+    if commit_id == "":
+        latest_commit = branch.commits.items[0]
+        choosen_commit_id = latest_commit.id
+        commit = client.commit.get(stream_id, choosen_commit_id)
+        print("latest commit ", branch.commits.items[0].createdAt, " was choosen")
+    elif type(commit_id) == type("s"): # string, commit uuid
+        choosen_commit_id = commit_id
+        commit = client.commit.get(stream_id, choosen_commit_id)
+        print("provided commit ", choosen_commit_id, " was choosen")
+    elif type(commit_id) == type(1): #int 
+        latest_commit = branch.commits.items[commit_id]
+        choosen_commit_id = latest_commit.id
+        commit = client.commit.get(stream_id, choosen_commit_id)
+
+
+    print(commit)
+    print(commit.referencedObject)
+    # get transport
+    transport = ServerTransport(client=client, stream_id=stream_id)
+    #speckle stream
+    res = operations.receive(commit.referencedObject, transport)
+
+    return res