modify pages

pages/1_🌲_Japan_Vegetation_Cover.py

@@ -1,14 +1,12 @@
-import datetime
-import os
+import streamlit as st
+import geopandas as gpd
+import pydeck as pdk
+import pandas as pd
+from branca import colormap as cm
 import pathlib
+import os
 import requests
 import zipfile
-import pandas as pd
-import pydeck as pdk
-import geopandas as gpd
-import streamlit as st
-import leafmap.colormaps as cm
-from leafmap.common import hex_to_rgb
 
 st.set_page_config(layout="wide")
 
@@ -27,197 +25,31 @@ st.sidebar.info(
 )
 
 STREAMLIT_STATIC_PATH = pathlib.Path(st.__path__[0]) / "static"
-# We create a downloads directory within the streamlit static asset directory
-# and we write output files to it
 DOWNLOADS_PATH = STREAMLIT_STATIC_PATH / "downloads"
 if not DOWNLOADS_PATH.is_dir():
     DOWNLOADS_PATH.mkdir()
 
-# Data source: https://www.realtor.com/research/data/
-# link_prefix = "https://econdata.s3-us-west-2.amazonaws.com/Reports/"
-link_prefix = "https://raw.githubusercontent.com/giswqs/data/main/housing/"
-
+# Data source
 data_links = {
-    "Chiba": {
-        "national": link_prefix + "Core/listing_weekly_core_aggregate_by_country.csv",
-        "metro": link_prefix + "Core/listing_weekly_core_aggregate_by_metro.csv",
-    },
-    "monthly_current": {
-        "national": link_prefix + "Core/RDC_Inventory_Core_Metrics_Country.csv",
-        "state": link_prefix + "Core/RDC_Inventory_Core_Metrics_State.csv",
-        "metro": link_prefix + "Core/RDC_Inventory_Core_Metrics_Metro.csv",
-        "county": link_prefix + "Core/RDC_Inventory_Core_Metrics_County.csv",
-        "zip": link_prefix + "Core/RDC_Inventory_Core_Metrics_Zip.csv",
-    },
-    "monthly_historical": {
-        "national": link_prefix + "Core/RDC_Inventory_Core_Metrics_Country_History.csv",
-        "state": link_prefix + "Core/RDC_Inventory_Core_Metrics_State_History.csv",
-        "metro": link_prefix + "Core/RDC_Inventory_Core_Metrics_Metro_History.csv",
-        "county": link_prefix + "Core/RDC_Inventory_Core_Metrics_County_History.csv",
-        "zip": link_prefix + "Core/RDC_Inventory_Core_Metrics_Zip_History.csv",
-    },
-    "hotness": {
-        "metro": link_prefix
-        + "Hotness/RDC_Inventory_Hotness_Metrics_Metro_History.csv",
-        "county": link_prefix
-        + "Hotness/RDC_Inventory_Hotness_Metrics_County_History.csv",
-        "zip": link_prefix + "Hotness/RDC_Inventory_Hotness_Metrics_Zip_History.csv",
-    },
+    "Tokyo": "https://github.com/kunifujiwara/data/blob/master/frac_veg/FRAC_VEG_Tokyo.geojson",
+    "Kanagawa": "https://github.com/kunifujiwara/data/blob/master/frac_veg/FRAC_VEG_Kanagawa.geojson",
+    "Chiba": "https://github.com/kunifujiwara/data/blob/master/frac_veg/FRAC_VEG_Chiba.geojson",
+    "Saitama": "https://github.com/kunifujiwara/data/blob/master/frac_veg/FRAC_VEG_Saitama.geojson",
 }
 
-
-def get_data_columns(df, category, frequency="monthly"):
-    if frequency == "monthly":
-        if category.lower() == "county":
-            del_cols = ["month_date_yyyymm", "county_fips", "county_name"]
-        elif category.lower() == "state":
-            del_cols = ["month_date_yyyymm", "state", "state_id"]
-        elif category.lower() == "national":
-            del_cols = ["month_date_yyyymm", "country"]
-        elif category.lower() == "metro":
-            del_cols = ["month_date_yyyymm", "cbsa_code", "cbsa_title", "HouseholdRank"]
-        elif category.lower() == "zip":
-            del_cols = ["month_date_yyyymm", "postal_code", "zip_name", "flag"]
-    elif frequency == "weekly":
-        if category.lower() == "national":
-            del_cols = ["week_end_date", "geo_country"]
-        elif category.lower() == "metro":
-            del_cols = ["week_end_date", "cbsa_code", "cbsa_title", "hh_rank"]
-
-    cols = df.columns.values.tolist()
-
-    for col in cols:
-        if col.strip() in del_cols:
-            cols.remove(col)
-    if category.lower() == "metro":
-        return cols[2:]
-    else:
-        return cols[1:]
-
-
-@st.cache_data
-def get_inventory_data(url):
-    df = pd.read_csv(url)
-    url = url.lower()
-    if "county" in url:
-        df["county_fips"] = df["county_fips"].map(str)
-        df["county_fips"] = df["county_fips"].str.zfill(5)
-    elif "state" in url:
-        df["STUSPS"] = df["state_id"].str.upper()
-    elif "metro" in url:
-        df["cbsa_code"] = df["cbsa_code"].map(str)
-    elif "zip" in url:
-        df["postal_code"] = df["postal_code"].map(str)
-        df["postal_code"] = df["postal_code"].str.zfill(5)
-
-    if "listing_weekly_core_aggregate_by_country" in url:
-        columns = get_data_columns(df, "national", "weekly")
-        for column in columns:
-            if column != "median_days_on_market_by_day_yy":
-                df[column] = df[column].str.rstrip("%").astype(float) / 100
-    if "listing_weekly_core_aggregate_by_metro" in url:
-        columns = get_data_columns(df, "metro", "weekly")
-        for column in columns:
-            if column != "median_days_on_market_by_day_yy":
-                df[column] = df[column].str.rstrip("%").astype(float) / 100
-        df["cbsa_code"] = df["cbsa_code"].str[:5]
-    return df
-
-
-def filter_weekly_inventory(df, week):
-    df = df[df["week_end_date"] == week]
-    return df
-
-
-def get_start_end_year(df):
-    start_year = int(str(df["month_date_yyyymm"].min())[:4])
-    end_year = int(str(df["month_date_yyyymm"].max())[:4])
-    return start_year, end_year
-
-
-def get_periods(df):
-    return [str(d) for d in list(set(df["month_date_yyyymm"].tolist()))]
-
-
 @st.cache_data
-def get_geom_data(category):
-
-    prefix = (
-        "https://raw.githubusercontent.com/giswqs/streamlit-geospatial/master/data/"
-    )
-    links = {
-        "national": prefix + "us_nation.geojson",
-        "state": prefix + "us_states.geojson",
-        "county": prefix + "us_counties.geojson",
-        "metro": prefix + "us_metro_areas.geojson",
-        "zip": "https://www2.census.gov/geo/tiger/GENZ2018/shp/cb_2018_us_zcta510_500k.zip",
-    }
-
-    if category.lower() == "zip":
-        r = requests.get(links[category])
-        out_zip = os.path.join(DOWNLOADS_PATH, "cb_2018_us_zcta510_500k.zip")
-        with open(out_zip, "wb") as code:
-            code.write(r.content)
-        zip_ref = zipfile.ZipFile(out_zip, "r")
-        zip_ref.extractall(DOWNLOADS_PATH)
-        gdf = gpd.read_file(out_zip.replace("zip", "shp"))
-    else:
-        gdf = gpd.read_file(links[category])
+def get_geom_data(prefecture):
+    gdf = gpd.read_file(data_links[prefecture])
     return gdf
 
-
-def join_attributes(gdf, df, category):
-
-    new_gdf = None
-    if category == "county":
-        new_gdf = gdf.merge(df, left_on="GEOID", right_on="county_fips", how="outer")
-    elif category == "state":
-        new_gdf = gdf.merge(df, left_on="STUSPS", right_on="STUSPS", how="outer")
-    elif category == "national":
-        if "geo_country" in df.columns.values.tolist():
-            df["country"] = None
-            df.loc[0, "country"] = "United States"
-        new_gdf = gdf.merge(df, left_on="NAME", right_on="country", how="outer")
-    elif category == "metro":
-        new_gdf = gdf.merge(df, left_on="CBSAFP", right_on="cbsa_code", how="outer")
-    elif category == "zip":
-        new_gdf = gdf.merge(df, left_on="GEOID10", right_on="postal_code", how="outer")
-    return new_gdf
-
+def get_data_columns(gdf):
+    return gdf.select_dtypes(include=[float, int]).columns.tolist()
 
 def select_non_null(gdf, col_name):
-    new_gdf = gdf[~gdf[col_name].isna()]
-    return new_gdf
-
+    return gdf[~gdf[col_name].isna()]
 
 def select_null(gdf, col_name):
-    new_gdf = gdf[gdf[col_name].isna()]
-    return new_gdf
-
-
-def get_data_dict(name):
-    in_csv = os.path.join(os.getcwd(), "data/realtor_data_dict.csv")
-    df = pd.read_csv(in_csv)
-    label = list(df[df["Name"] == name]["Label"])[0]
-    desc = list(df[df["Name"] == name]["Description"])[0]
-    return label, desc
-
-
-def get_weeks(df):
-    seq = list(set(df[~df["week_end_date"].isnull()]["week_end_date"].tolist()))
-    weeks = [
-        datetime.date(int(d.split("/")[2]), int(d.split("/")[0]), int(d.split("/")[1]))
-        for d in seq
-    ]
-    weeks.sort()
-    return weeks
-
-
-def get_saturday(in_date):
-    idx = (in_date.weekday() + 1) % 7
-    sat = in_date + datetime.timedelta(6 - idx)
-    return sat
-
+    return gdf[gdf[col_name].isna()]
 
 def app():
     st.title("Japan Vegetation Cover Fraction")
@@ -225,18 +57,14 @@ def app():
         """**Introduction:** This interactive dashboard is designed for visualizing Japan Fractional Vegetation Cover at town block levels. 
         The data sources include [Vegetation Cover Fraction](https://zenodo.org/records/5553516) from a research project (https://doi.org/10.3130/aijt.28.521), 
         and [Cartographic Boundary Files](https://www.e-stat.go.jp/gis/statmap-search?page=1&type=2&aggregateUnitForBoundary=A&toukeiCode=00200521&toukeiYear=2015&serveyId=A002005212015&coordsys=1&format=shape&datum=2000) from Census of Japan 2015.
-         Several open-source packages are used to process the data and generate the visualizations, e.g., [streamlit](https://streamlit.io),
-          [geopandas](https://geopandas.org), [leafmap](https://leafmap.org), and [pydeck](https://deckgl.readthedocs.io).
-    """
+        """
     )
 
     prefecture = st.selectbox("Prefecture", ["Tokyo", "Kanagawa", "Chiba", "Saitama"])
     
-    # Load GeoJSON data
-    gdf = gpd.read_file(f'https://github.com/kunifujiwara/data/blob/master/frac_veg/FRAC_VEG_{prefecture}.geojson')
+    gdf = get_geom_data(prefecture)
 
-    # Select attribute to visualize
-    attributes = gdf.select_dtypes(include=[float, int]).columns.tolist()
+    attributes = get_data_columns(gdf)
     selected_attribute = st.selectbox("Select attribute to visualize", attributes)
 
     row2_col1, row2_col2, row2_col3, row2_col4, row2_col5, row2_col6 = st.columns(
@@ -262,14 +90,13 @@ def app():
         else:
             elev_scale = 1
 
-    # Create color map
     color_scale = cm.LinearColormap(colors=cm.get_palette(palette, n_colors), vmin=gdf[selected_attribute].min(), vmax=gdf[selected_attribute].max())
     gdf['color'] = gdf[selected_attribute].apply(lambda x: color_scale(x))
-
-    # Convert hex colors to RGB
     gdf['color'] = gdf['color'].apply(lambda x: [int(x[1:3], 16), int(x[3:5], 16), int(x[5:7], 16)])
 
-    # Create PyDeck layer
+    gdf_null = select_null(gdf, selected_attribute)
+    gdf = select_non_null(gdf, selected_attribute)
+
     layer = pdk.Layer(
         "GeoJsonLayer",
         gdf,
@@ -287,7 +114,24 @@ def app():
         line_width_min_pixels=1,
     )
 
-    # Set initial view state
+    if show_nodata:
+        nodata_layer = pdk.Layer(
+            "GeoJsonLayer",
+            gdf_null,
+            pickable=True,
+            opacity=0.2,
+            stroked=True,
+            filled=True,
+            extruded=False,
+            get_fill_color=[200, 200, 200],
+            get_line_color=[0, 0, 0],
+            get_line_width=2,
+            line_width_min_pixels=1,
+        )
+        layers = [layer, nodata_layer]
+    else:
+        layers = [layer]
+
     view_state = pdk.ViewState(
         latitude=gdf.geometry.centroid.y.mean(),
         longitude=gdf.geometry.centroid.x.mean(),
@@ -295,23 +139,19 @@ def app():
         pitch=45 if show_3d else 0,
     )
 
-    # Create PyDeck chart
     r = pdk.Deck(
-        layers=[layer],
+        layers=layers,
         initial_view_state=view_state,
         map_style="mapbox://styles/mapbox/light-v9",
         tooltip={"text": "{NAME}\n{" + selected_attribute + "}"}
     )
 
-    # Display the map
     st.pydeck_chart(r)
 
-    # Display color scale
     st.write(color_scale)
 
-    # Option to show raw data
     if st.checkbox("Show raw data"):
         st.write(gdf[[selected_attribute, 'NAME']])
 
-
-app()
+if __name__ == "__main__":
+    app()