diff --git "a/preprocess-redlining.ipynb" "b/preprocess-redlining.ipynb"
new file mode 100644--- /dev/null
+++ "b/preprocess-redlining.ipynb"
@@ -0,0 +1,464 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "5fcd59bc-72a4-4de7-9cdb-1b6eca9407fb",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "2024-12-02 22:03:04.459 No runtime found, using MemoryCacheStorageManager\n"
+ ]
+ }
+ ],
+ "source": [
+ "import ibis\n",
+ "from ibis import _\n",
+ "import streamlit as st\n",
+ "\n",
+ "con = ibis.duckdb.connect(\"duck.db\", extensions=['httpfs', 'spatial', 'h3'])\n",
+ "\n",
+ "# Local cloud\n",
+ "minio_key = st.secrets[\"MINIO_KEY\"]\n",
+ "minio_secret = st.secrets[\"MINIO_SECRET\"]\n",
+ "query= f'''\n",
+ "CREATE OR REPLACE SECRET secret2 (\n",
+ " TYPE S3,\n",
+ " KEY_ID '{minio_key}',\n",
+ " SECRET '{minio_secret}',\n",
+ " ENDPOINT 'minio.carlboettiger.info',\n",
+ " URL_STYLE 'path',\n",
+ " SCOPE 's3://cboettig/gbif'\n",
+ ");\n",
+ "'''\n",
+ "con.raw_sql(query)\n",
+ "\n",
+ "## Limits are sometimes good \n",
+ "#con.raw_sql(\"SET memory_limit = '20GB';\")\n",
+ "#con.raw_sql(\"set threads=40;\")\n",
+ "\n",
+ "gbif = con.read_parquet(\"s3://cboettig/gbif/2024-10-01/**\")\n",
+ "\n",
+ "# can/should we add explicit spatial index to gbif first? using RTree takes too much memory"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2e1f1ee1-8048-4f2b-83a0-ca8cc8fc9067",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Copy to local cloud as parquet\n",
+ "\n",
+ "(con\n",
+ " .read_geo(\"/vsicurl/https://data.source.coop/cboettig/us-boundaries/mappinginequality.json\")\n",
+ " .to_parquet(\"s3://cboettig/gbif/mappinginequality.parquet\")\n",
+ ")\n",
+ "\n",
+ "# can/should we add explicit spatial index to mappinginequality polygons first? \n",
+ "# would local duckdb table version be even better/faster? \n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3891abb6-3652-4217-8615-106d354ff131",
+ "metadata": {},
+ "source": [
+ "We iterate through the city list to do this efficiently. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "id": "32a2b4c1-e08b-4fbb-b891-ac19053a4585",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "## select cities from the list we haven't yet written (allows resume).\n",
+ "import minio\n",
+ "import re\n",
+ "\n",
+ "minio_key = st.secrets[\"MINIO_KEY\"]\n",
+ "minio_secret = st.secrets[\"MINIO_SECRET\"]\n",
+ "mc = minio.Minio(\"minio.carlboettiger.info\", minio_key, minio_secret)\n",
+ "obj = mc.list_objects(\"cboettig\", \"gbif/redlined/\", recursive=True)\n",
+ "\n",
+ "pattern = r\"gbif/redlined/|\\.parquet$\"\n",
+ "finished_cities = [re.sub(pattern, \"\", i.object_name) for i in obj if not i.is_dir]\n",
+ "\n",
+ "mappinginequality = con.read_parquet(\"s3://cboettig/gbif/mappinginequality.parquet\")\n",
+ "all_cities = (mappinginequality.select(_.city, _.state)\n",
+ " .distinct()\n",
+ " .mutate(index = \"state=\" + _.state + \"/\" + \"city=\" + _.city)\n",
+ " .execute()[\"index\"]\n",
+ " )\n",
+ "\n",
+ "remaining_cities = set(all_cities) - set(finished_cities)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "id": "58e4a062-8f7e-45d2-a41f-fdc49b1c9f03",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'state=SC/city=Sumter'}"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(remaining_cities)\n",
+ "remaining_cities"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "id": "c3a4005c-1e8c-4f2a-a93c-1c158c9c26ab",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "state=SC/city=Sumter\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "9dee1bc3c5c34370892812ff0894d8e4",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 15min 54s, sys: 4min 38s, total: 20min 33s\n",
+ "Wall time: 48.8 s\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%time \n",
+ "\n",
+ "## And here we go, long-running loop over each city\n",
+ "for i in remaining_cities:\n",
+ " gdf = (mappinginequality\n",
+ " .mutate(index = \"state=\" + _.state + \"/\" + \"city=\" + _.city)\n",
+ " .filter(_.index == i)\n",
+ " .mutate(area = _.geom.area())\n",
+ " )\n",
+ "\n",
+ " print(i)\n",
+ " \n",
+ " bounds = gdf.execute().total_bounds\n",
+ " points = (gbif\n",
+ " .filter(_.decimallongitude >= bounds[0], \n",
+ " _.decimallongitude < bounds[2], \n",
+ " _.decimallatitude >= bounds[1], \n",
+ " _.decimallatitude < bounds[3])\n",
+ " )\n",
+ " \n",
+ " (gdf\n",
+ " .join(points, gdf.geom.intersects(points.geom))\n",
+ " .to_parquet(f\"s3://cboettig/gbif/redlined/{i}.parquet\")\n",
+ " )\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7e0ed68d-e6ba-44c8-b7a0-1f284b11cb96",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 89,
+ "id": "976eb48e-eaf1-45db-9034-84ac33eeb506",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "5bed6c7dc2ae4962994a4bf8920d12fb",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# standardize grades and consolidate to single parquet?\n",
+ "\n",
+ "(con.read_parquet(\"s3://cboettig/gbif/redlined/**\")\n",
+ " .mutate(grade=_.grade.strip())\n",
+ " .mutate(grade=_.grade.cases((('A', 'A'), ('B', 'B'), ('C', 'C'), ('D', 'D')), default = \"None\" ) )\n",
+ " .to_parquet(\"s3://cboettig/gbif/redlined_cities_gbif.parquet\")\n",
+ ")\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 96,
+ "id": "5ce74a60-c30b-4a78-9653-b34e788227b3",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 2.59 s, sys: 22.9 s, total: 25.5 s\n",
+ "Wall time: 700 ms\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " grade | \n",
+ " n | \n",
+ " area | \n",
+ " density | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " A | \n",
+ " 3048483 | \n",
+ " 1008.034173 | \n",
+ " 3024.186167 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " B | \n",
+ " 5506468 | \n",
+ " 2517.018555 | \n",
+ " 2187.694639 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " D | \n",
+ " 3491760 | \n",
+ " 1896.670719 | \n",
+ " 1840.994309 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " C | \n",
+ " 6353620 | \n",
+ " 3539.963360 | \n",
+ " 1794.826487 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " None | \n",
+ " 6021724 | \n",
+ " 16665.329460 | \n",
+ " 361.332431 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "