:broom:

app.R

@@ -1,20 +1,23 @@
 library(shiny)
 library(bslib)
 library(htmltools)
-library(markdown)
+#library(markdown)
 library(fontawesome)
 library(bsicons)
 library(gt)
 library(glue)
 library(ggplot2)
-
-library(mapgl)
+library(readr)
 library(dplyr)
+library(mapgl)
 library(duckdbfs)
-
 duckdbfs::load_spatial()
 
-css <- HTML("<link rel='stylesheet' type='text/css' href='https://demos.creative-tim.com/material-dashboard/assets/css/material-dashboard.min.css?v=3.2.0'>")
+css <-
+  HTML(paste0("<link rel='stylesheet' type='text/css' ",
+              "href='https://demos.creative-tim.com/",
+              "material-dashboard/assets/css/",
+              "material-dashboard.min.css?v=3.2.0'>"))
 
 
 # Define the UI
@@ -23,22 +26,23 @@ ui <- page_sidebar(
   tags$head(css),
   titlePanel("Demo App"),
 
-"This is a proof-of-principle for a simple chat-driven interface to dynamically explore geospatial data. 
- ",
-
+  "
+  This is a proof-of-principle for a simple chat-driven interface
+  to dynamically explore geospatial data.
+  ",
 
   card(
     layout_columns(
-    textInput("chat",
-      label = NULL,
-      "Which counties in California have the highest average social vulnerability?",
-      width = "100%"),
-    div(
-    actionButton("user_msg", "", icon = icon("paper-plane"),
-                 class = "btn-primary btn-sm align-bottom"),
-    class = "align-text-bottom"),
-    col_widths = c(11, 1)),
-    fill = FALSE
+      textInput("chat",
+        label = NULL,
+        "Which counties in California have the highest average social vulnerability?",
+        width = "100%"),
+      div(
+      actionButton("user_msg", "", icon = icon("paper-plane"),
+                   class = "btn-primary btn-sm align-bottom"),
+      class = "align-text-bottom"),
+      col_widths = c(11, 1)),
+      fill = FALSE
   ),
   layout_columns(
     card(maplibreOutput("map")),
@@ -51,12 +55,10 @@ ui <- page_sidebar(
     max_height = "700px"
   ),
 
-
   gt_output("table"),
 
   card(fill = TRUE,
     card_header(fa("robot")),
- 
     accordion(
       open = FALSE,
       accordion_panel(
@@ -70,34 +72,13 @@ ui <- page_sidebar(
         textOutput("explanation"),
       )
     ),
-
     card(
       card_header("Errata"),
-      markdown(
-"
-#### Credits
-
-Developed by Carl Boettiger, UC Berkeley, 2025.  BSD License.
-
-Data from the US Census and CDC's [Social Vulnerability Index](https://www.atsdr.cdc.gov/place-health/php/svi/index.html)
-
-#### Technical details
-
-The app is written entirely in R using shiny. The app will translate natural language queries in SQL code using
-a small open-weights language model. The SQL code is executed using the duckdb backend against cloud-native
-geoparquet snapshot of the Social Vulnerability Index hosted on Source Cooperative. Summary chart data are also
-computed in duckdb by streaming, providing responsive updates while needing minimal RAM or disk storage despite
-the large size of the data sources. 
-
-The map is rendered and updated using MapLibre with PMTiles, which provides responsive rendering for large feature sets.
-The PMTiles layer is also hosted on Source cooperative where it can be streamed efficiently.
-")
+      shiny::markdown(readr::read_file("footer.md")),
     )
-
   ),
 
   sidebar = sidebar(
-
     input_switch("redlines", "Redlined Areas", value = FALSE),
     input_switch("svi", "Social Vulnerability", value = TRUE),
     input_switch("richness", "Biodiversity Richness", value = FALSE),
@@ -113,40 +94,14 @@ The PMTiles layer is also hosted on Source cooperative where it can be streamed
 )
 
 
-
-
 repo <- "https://data.source.coop/cboettig/social-vulnerability"
 pmtiles <- glue("{repo}/svi2020_us_tract.pmtiles")
 parquet <- glue("{repo}/svi2020_us_tract.parquet")
-svi <- open_dataset(parquet, tblname = "svi") |>
-  filter(RPL_THEMES > 0)
-
-
 con <- duckdbfs::cached_connection()
-schema <- DBI::dbGetQuery(con, "PRAGMA table_info(svi)")
-
-system_prompt = glue::glue('
-You are a helpful agent who always replies strictly in JSON-formatted text.
-Your task is to translate the users question into a SQL query that will be run
-against the "svi" table in a duckdb database. The duckdb database has a
-spatial extension which understands PostGIS operations as well. 
-Include semantically meaningful columns like COUNTY and STATE name.
-
-In the data, each row represents an individual census tract. If asked for 
-county or state level statistics, be sure to aggregate across all the tracts
-in that county or state. 
-
-The table schema is <schema>
-
-The column called "RPL_THEMES" corresponds to the overall "Social vulnerability index" number. 
-
-Format your answer as follows:
-
-{
-"query": "your raw SQL response goes here",
-"explanation": "your explanation of the query"
-}
-', .open = "<", .close = ">")
+svi <- open_dataset(parquet, tblname = "svi") |> filter(RPL_THEMES > 0)
+schema <- read_file("schema.yml")
+system_prompt <- glue::glue(readr::read_file("system-prompt.md"),
+                            .open = "<", .close = ">")
 
 chat <- ellmer::chat_vllm(
   base_url = "https://llm.nrp-nautilus.io/",
@@ -168,18 +123,20 @@ filter_column <- function(full_data, filtered_data, id_col = "FIPS") {
   list("in", list("get", id_col), list("literal", values))
 }
 
-chart1_data <- svi |>
-  group_by(COUNTY) |>
-  summarise(mean_svi = mean(RPL_THEMES)) |>
-  collect()
-
-chart1 <- chart1_data |>
-  ggplot(aes(mean_svi)) + geom_density(fill="darkred") +
-  ggtitle("County-level vulnerability nation-wide")
 
 
 # Define the server
 server <- function(input, output, session) {
+
+  chart1_data <- svi |>
+    group_by(COUNTY) |>
+    summarise(mean_svi = mean(RPL_THEMES)) |>
+    collect()
+
+  chart1 <- chart1_data |>
+    ggplot(aes(mean_svi)) + geom_density(fill="darkred") +
+    ggtitle("County-level vulnerability nation-wide")
+
   data <- reactiveValues(df = tibble())
   output$chart1 <- renderPlot(chart1)
 
@@ -191,29 +148,32 @@ server <- function(input, output, session) {
 
     # Parse response
     response <- jsonlite::fromJSON(stream)
-    output$sql_code <- renderText(stringr::str_wrap(response$query, width = 60))
-    output$explanation <- renderText(response$explanation)
 
-    # Actually execute the SQL query generated:
-    df <- DBI::dbGetQuery(con, response$query)
+    if ("query" %in% names(response)) {
+      output$sql_code <- renderText(stringr::str_wrap(response$query, width = 60))
+      output$explanation <- renderText(response$explanation)
+
+      # Actually execute the SQL query generated:
+      df <- DBI::dbGetQuery(con, response$query)
 
-    # don't display shape column in render
-    df <- df |> select(-any_of("Shape"))
-    output$table <- render_gt(df, height = 300)
+      # don't display shape column in render
+      df <- df |> select(-any_of("Shape"))
+      output$table <- render_gt(df, height = 300)
 
 
-    y_axis <- colnames(df)[!colnames(df) %in% colnames(svi)] 
-    chart2 <- df |> 
-      rename(social_vulnerability = y_axis) |>
-      ggplot(aes(social_vulnerability)) +
-      geom_density(fill = "darkred")  +
-      xlim(c(0, 1)) +
-      ggtitle("Vulnerability of selected areas")
+      y_axis <- colnames(df)[!colnames(df) %in% colnames(svi)]
+      chart2 <- df |> 
+        rename(social_vulnerability = y_axis) |>
+        ggplot(aes(social_vulnerability)) +
+        geom_density(fill = "darkred")  +
+        xlim(c(0, 1)) +
+        ggtitle("Vulnerability of selected areas")
 
-    output$chart2 <- renderPlot(chart2)
+      output$chart2 <- renderPlot(chart2)
 
-    # We need to somehow trigger this df to update the map.
-    data$df <- df
+      # We need to somehow trigger this df to update the map.
+      data$df <- df
+    }
 
   })
 

footer.md

@@ -0,0 +1,16 @@
+#### Credits
+
+Developed by Carl Boettiger, UC Berkeley, 2025.  BSD License.
+
+Data from the US Census and CDC's [Social Vulnerability Index](https://www.atsdr.cdc.gov/place-health/php/svi/index.html)
+
+#### Technical details
+
+The app is written entirely in R using shiny. The app will translate natural language queries in SQL code using
+a small open-weights language model. The SQL code is executed using the duckdb backend against cloud-native
+geoparquet snapshot of the Social Vulnerability Index hosted on Source Cooperative. Summary chart data are also
+computed in duckdb by streaming, providing responsive updates while needing minimal RAM or disk storage despite
+the large size of the data sources. 
+
+The map is rendered and updated using MapLibre with PMTiles, which provides responsive rendering for large feature sets.
+The PMTiles layer is also hosted on Source cooperative where it can be streamed efficiently.

schema.yml

@@ -0,0 +1,26 @@
+- VARIABLE_NAME: ST
+  DESCRIPTION: State-level FIPS code (two-digit integer)
+- VARIABLE_NAME: STATE
+  DESCRIPTION: State name
+- VARIABLE_NAME: ST_ABBR
+  DESCRIPTION: State abbreviation
+- VARIABLE_NAME: STCNTY
+  DESCRIPTION: County-level FIPS code (5 digit integer)
+- VARIABLE_NAME: COUNTY
+  DESCRIPTION: County name
+- VARIABLE_NAME: FIPS
+  DESCRIPTION: Tract-level geographic identification (full Census Bureau FIPS code)
+- VARIABLE_NAME: LOCATION
+  DESCRIPTION: Text description of tract county state
+- VARIABLE_NAME: AREA_SQMI
+  DESCRIPTION: Tract area in square miles
+- VARIABLE_NAME: RPL_THEMES
+  DESCRIPTION: Overall social vulnerability. Should always be used unless explicit sub-theme is called for.
+- VARIABLE_NAME: RPL_THEME1
+  DESCRIPTION: Subtheme for socio-economic status social vulnerability score
+- VARIABLE_NAME: RPL_THEME2
+  DESCRIPTION: Subtheme for Household characteristics vulnerability score
+- VARIABLE_NAME: RPL_THEME3
+  DESCRIPTION: Subtheme for Racial and Ethnic Minority status based vulnerability score
+- VARIABLE_NAME: RPL_THEME4
+  DESCRIPTION: Subtheme for Housing and transportation-based vulnerability score

system-prompt.md

@@ -0,0 +1,34 @@
+
+You are a helpful agent who always replies strictly in JSON-formatted text.
+Your task is to translate the user's questions about the data into a SQL query
+that will be run against the "svi" table in a duckdb database.
+The duckdb database has a spatial extension which understands PostGIS operations as well.
+Include semantically meaningful columns like COUNTY and STATE name.
+
+If your answer involves the construction of a SQL query, you must format your answer as follows:
+
+{
+"query": "your raw SQL response goes here",
+"explanation": "your explanation of the query"
+}
+
+If your answer does not involve a SQL query, please reply with the following format instead:
+
+{
+    "user": "user question goes here",
+    "agent": "your response goes here"
+}
+
+If you are asked to describe the data or for information about the data schema, give only a human-readable response with SQL.
+
+In the data, each row represents an individual census tract. If asked for 
+county or state level statistics, be sure to aggregate across all the tracts
+in that county or state. 
+
+Refer to this descriptions of each of the columns (VARIABLE_NAME) from the metadata table:
+<schema>
+
+Note that the column called "RPL_THEMES" corresponds to the overall "Social vulnerability index" number. Whenenver you SELECT the COUNTY, you must include the STCNTY
+column as well because county names are not unique across states!
+
+

test.R

@@ -0,0 +1,71 @@
+## Illustrate/test core app functionality without shiny
+
+library(tidyverse)
+library(duckdbfs)
+library(mapgl)
+library(ellmer)
+library(glue)
+
+repo <- "https://data.source.coop/cboettig/social-vulnerability"
+pmtiles <- glue("{repo}/svi2020_us_tract.pmtiles")
+parquet <- glue("{repo}/svi2020_us_tract.parquet")
+svi <- open_dataset(parquet, tblname = "svi") |> filter(RPL_THEMES > 0)
+
+schema <- read_file("schema.yml")
+system_prompt <- glue::glue(readr::read_file("system-prompt.md"),
+                            .open = "<", .close = ">")
+
+chat <- ellmer::chat_vllm(
+  base_url = "https://llm.nrp-nautilus.io/",
+  model = "llama3",
+  api_key = Sys.getenv("NRP_API_KEY"),
+  system_prompt = system_prompt,
+  api_args = list(temperature = 0)
+)
+
+chat <- ellmer::chat_vllm(
+  base_url = "https://llm.cirrus.carlboettiger.info/v1/",
+  model = "kosbu/Llama-3.3-70B-Instruct-AWQ",
+  api_key = Sys.getenv("CIRRUS_LLM_KEY"),
+  system_prompt = system_prompt,
+  api_args = list(temperature = 0)
+)
+
+# Test a chat-based response
+chat$chat("Which columns describes racial components of social vulnerability?")
+## A query-based response
+stream <- chat$chat("Which counties in California have the highest average social vulnerability?")
+response <- jsonlite::fromJSON(stream)
+
+con <- duckdbfs::cached_connection()
+filtered_data <- DBI::dbGetQuery(con, response$query)
+full_data <- svi
+response_query <- "
+SELECT COUNTY, AVG(RPL_THEME1) as avg_soc_vuln FROM
+svi WHERE STATE = 'California' GROUP BY COUNTY ORDER BY
+avg_soc_vuln DESC LIMIT 10;
+"
+
+
+filter_column <- function(full_data, filtered_data, id_col) {
+  if (nrow(filtered_data) < 1) return(NULL)
+  values <- full_data |>
+    inner_join(filtered_data, copy = TRUE) |>
+    pull(id_col)
+  # maplibre syntax for the filter of PMTiles  
+  list("in", list("get", id_col), list("literal", values))
+}
+
+maplibre(center = c(-102.9, 41.3), zoom = 3) |>
+    add_fill_layer(
+        id = "svi_layer",
+        source = list(type = "vector", url  = paste0("pmtiles://", pmtiles)),
+        source_layer = "SVI2000_US_tract",
+        filter = filter_column(full_data, filtered_data, "FIPS"),
+        fill_opacity = 0.5,
+        fill_color = interpolate(column = "RPL_THEMES",
+                                values = c(0, 1),
+                                stops = c("#e19292c0", "darkblue"),
+                                na_color = "lightgrey")
+    )
+