Upload 5 files

Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.11-slim
+
+WORKDIR /code
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create necessary directories with permissions
+RUN mkdir -p /.cache bluesky_data
+RUN chmod 777 /.cache bluesky_data
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Expose the port
+EXPOSE 7860
+
+# Set environment variables (these should be overridden at runtime)
+ENV HF_REPO_ID="davanstrien/bluesky-counts"
+ENV HF_REPO_TYPE="dataset"
+ENV HF_TOKEN=""
+
+CMD ["python", "app.py"]

README.md

@@ -1,10 +1,10 @@
 ---
-title: Bluesky Posts Over Time
-emoji: 💻
-colorFrom: pink
-colorTo: red
+title: Bluesky Post Counter
+emoji: 📊
+colorFrom: blue
+colorTo: indigo
 sdk: docker
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Real-time visualization of posts per second on Bluesky social network.

app.py

@@ -0,0 +1,167 @@
+import hvplot.streamz
+import pandas as pd
+import numpy as np
+from streamz import Stream
+from streamz.dataframe import DataFrame
+from atproto import FirehoseSubscribeReposClient, parse_subscribe_repos_message
+import datetime
+import queue
+import threading
+import time
+import os
+import json
+from huggingface_hub import CommitScheduler, HfApi, hf_hub_download
+import uuid
+from pathlib import Path
+import panel as pn
+
+
+pn.extension(design="material")
+# Create a queue to communicate between threads
+post_queue = queue.Queue()
+
+# Counter for posts
+post_count = 0
+
+# Create streaming dataframe
+stream = Stream()
+# Wait 1 second to collect initial data
+time.sleep(1)
+example = pd.DataFrame(
+    {"timestamp": [pd.Timestamp.now()], "post_count": [post_count]}, index=[0]
+)
+df = DataFrame(stream, example=example)
+
+# Calculate backlog for 1 month (31 days)
+MONTH_IN_SECONDS = 31 * 24 * 60 * 60  # 31 days * 24 hours * 60 minutes * 60 seconds
+
+# Add environment variable support for configuration
+REPO_ID = os.getenv("HF_REPO_ID", "davanstrien/bluesky-counts")
+REPO_TYPE = os.getenv("HF_REPO_TYPE", "dataset")
+HF_TOKEN = os.getenv("HUGGINGFACE_TOKEN")  # Required for HuggingFace API access
+DATA_FOLDER = Path("bluesky_data")
+DATA_FILE = f"bluesky_counts_{uuid.uuid4()}.json"
+
+
+def load_hub_data():
+    """Load the most recent data from the Hub"""
+    try:
+        api = HfApi(token=HF_TOKEN)
+        # List files in the repository
+        files = api.list_repo_files(REPO_ID, repo_type=REPO_TYPE)
+        data_files = [f for f in files if f.startswith("data/bluesky_counts_")]
+
+        if not data_files:
+            return []
+
+        # Get the most recent file
+        latest_file = sorted(data_files)[-1]
+        # Download the file
+        local_path = hf_hub_download(
+            repo_id=REPO_ID, filename=latest_file, repo_type=REPO_TYPE, token=HF_TOKEN
+        )
+
+        # Load and parse the data
+        data = []
+        with open(local_path, "r") as f:
+            data.extend(json.loads(line.strip()) for line in f)
+        # Keep only last month of data
+        return data[-MONTH_IN_SECONDS:]
+    except Exception as e:
+        print(f"Error loading data from Hub: {e}")
+        return []
+
+
+# Initialize storage and Hub connection
+DATA_FOLDER.mkdir(exist_ok=True)
+scheduler = CommitScheduler(
+    repo_id=REPO_ID,
+    repo_type=REPO_TYPE,
+    folder_path=DATA_FOLDER,
+    path_in_repo="data",
+    every=600,  # Upload every 10 minutes
+    token=HF_TOKEN,  # Add token for authentication
+)
+
+
+def on_message_handler(message):
+    global post_count
+    commit = parse_subscribe_repos_message(message)
+    # Only count new posts (not likes, reposts, etc)
+    if hasattr(commit, "ops"):
+        for op in commit.ops:
+            if op.action == "create" and "app.bsky.feed.post" in op.path:
+                post_count += 1
+
+
+def emit_counts():
+    """Emit post counts every second"""
+    global post_count
+
+    if saved_data := load_hub_data():
+        print(f"Loaded {len(saved_data)} historical data points from Hub")
+        # Emit historical data
+        for point in saved_data[-100:]:  # Emit last 100 points to initialize plot
+            df = pd.DataFrame(
+                {
+                    "timestamp": [pd.Timestamp(point["timestamp"])],
+                    "post_count": [point["post_count"]],
+                }
+            )
+            stream.emit(df)
+
+    # Wait for first second to collect initial data
+    time.sleep(1)
+
+    while True:
+        # Create DataFrame with current timestamp and count
+        now = pd.Timestamp.now()
+        df = pd.DataFrame({"timestamp": [now], "post_count": [post_count]})
+        stream.emit(df)
+
+        # Reset counter
+        post_count = 0
+        # Wait 1 second
+        time.sleep(1)
+
+
+# Create the plot with month-long backlog
+plot = df.hvplot.line(
+    "timestamp",
+    "post_count",
+    title="Bluesky Posts per Second",
+    width=800,
+    height=400,
+    backlog=MONTH_IN_SECONDS,  # Keep last month of points
+)
+
+
+# Start Firehose client in a separate thread
+def run_firehose():
+    client = FirehoseSubscribeReposClient()
+    client.start(on_message_handler)
+
+
+firehose_thread = threading.Thread(target=run_firehose)
+firehose_thread.daemon = True
+firehose_thread.start()
+
+# Start emitting counts in another thread
+emit_thread = threading.Thread(target=emit_counts)
+emit_thread.daemon = True
+emit_thread.start()
+
+# If running in a Jupyter notebook, display the plot
+if __name__ == "__main__":
+    import panel as pn
+
+    pn.extension()
+    dashboard = pn.Column(pn.pane.HoloViews(plot))
+    # Update server configuration for Docker
+    pn.serve(
+        dashboard,
+        address="0.0.0.0",
+        port=7860,
+        allow_websocket_origin=["*"],  # Changed from "*" to ["*"]
+        show=False,
+    )

requirements.in

@@ -0,0 +1,4 @@
+panel
+hvplot
+streamz
+atproto

requirements.txt

@@ -0,0 +1,173 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements.in -o requirements.txt
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.6.2.post1
+    # via httpx
+atproto==0.0.55
+    # via -r requirements.in
+bleach==6.2.0
+    # via panel
+bokeh==3.6.1
+    # via
+    #   holoviews
+    #   hvplot
+    #   panel
+certifi==2024.8.30
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+cffi==1.17.1
+    # via cryptography
+charset-normalizer==3.4.0
+    # via requests
+click==8.1.7
+    # via atproto
+colorcet==3.1.0
+    # via
+    #   holoviews
+    #   hvplot
+contourpy==1.3.1
+    # via bokeh
+cryptography==43.0.3
+    # via atproto
+dnspython==2.7.0
+    # via atproto
+filelock==3.16.1
+    # via huggingface-hub
+fsspec==2024.10.0
+    # via huggingface-hub
+h11==0.14.0
+    # via httpcore
+holoviews==1.20.0
+    # via hvplot
+httpcore==1.0.7
+    # via httpx
+httpx==0.27.2
+    # via atproto
+huggingface-hub==0.26.2
+    # via -r requirements.in
+hvplot==0.11.1
+    # via -r requirements.in
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+jinja2==3.1.4
+    # via bokeh
+libipld==3.0.0
+    # via atproto
+linkify-it-py==2.0.3
+    # via panel
+markdown==3.7
+    # via panel
+markdown-it-py==3.0.0
+    # via
+    #   mdit-py-plugins
+    #   panel
+markupsafe==3.0.2
+    # via jinja2
+mdit-py-plugins==0.4.2
+    # via panel
+mdurl==0.1.2
+    # via markdown-it-py
+numpy==2.1.3
+    # via
+    #   bokeh
+    #   contourpy
+    #   holoviews
+    #   hvplot
+    #   pandas
+packaging==24.2
+    # via
+    #   bokeh
+    #   holoviews
+    #   huggingface-hub
+    #   hvplot
+    #   panel
+pandas==2.2.3
+    # via
+    #   bokeh
+    #   holoviews
+    #   hvplot
+    #   panel
+panel==1.5.4
+    # via
+    #   -r requirements.in
+    #   holoviews
+    #   hvplot
+param==2.1.1
+    # via
+    #   holoviews
+    #   hvplot
+    #   panel
+    #   pyviz-comms
+pillow==11.0.0
+    # via bokeh
+pycparser==2.22
+    # via cffi
+pydantic==2.10.1
+    # via atproto
+pydantic-core==2.27.1
+    # via pydantic
+python-dateutil==2.9.0.post0
+    # via pandas
+pytz==2024.2
+    # via pandas
+pyviz-comms==3.0.3
+    # via
+    #   holoviews
+    #   panel
+pyyaml==6.0.2
+    # via
+    #   bokeh
+    #   huggingface-hub
+requests==2.32.3
+    # via
+    #   huggingface-hub
+    #   panel
+setuptools==75.6.0
+    # via streamz
+six==1.16.0
+    # via
+    #   python-dateutil
+    #   streamz
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
+streamz==0.6.4
+    # via -r requirements.in
+toolz==1.0.0
+    # via streamz
+tornado==6.4.2
+    # via
+    #   bokeh
+    #   streamz
+tqdm==4.67.1
+    # via
+    #   huggingface-hub
+    #   panel
+typing-extensions==4.12.2
+    # via
+    #   atproto
+    #   huggingface-hub
+    #   panel
+    #   pydantic
+    #   pydantic-core
+tzdata==2024.2
+    # via pandas
+uc-micro-py==1.0.3
+    # via linkify-it-py
+urllib3==2.2.3
+    # via requests
+webencodings==0.5.1
+    # via bleach
+websockets==13.1
+    # via atproto
+xyzservices==2024.9.0
+    # via bokeh
+zict==3.0.0
+    # via streamz