Add new CMAP data hub links

Add new links and adjust code to reflect new structure (JSON downloads rather than ZIP archives).
CMAP-REPOS · Dec 18, 2023 · fcc4361 · fcc4361
1 parent 7b5edd1
commit fcc4361
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 74 deletions.
diff --git a/data-raw/load_census_api.R b/data-raw/load_census_api.R
@@ -70,30 +70,48 @@ municipality_sf <- tigris::places(state = STATE, year = BASE_YEAR) %>%
   select(geoid_place, municipality, sqmi) %>%
   arrange(geoid_place)
 
-# Process Congressional Districts (U.S. House of Representatives) Code to
-# manually download congressional districts due to issue with tigris
-# functionality on congressional districts
+# Process Congressional Districts (U.S. House of Representatives)
 
-######################################
+################################################################
+#### Code to manually download congressional districts due to issue with tigris
+#### functionality on congressional districts. All lines involving downloading
+#### temp files and extracting can be removed when tigris is updated.
 
-download.file("https://www2.census.gov/geo/tiger/TIGER_RD18/LAYER/CD/tl_rd22_17_cd118.zip",
-              destfile = "cd_2022.zip")
-unzip("cd_2022.zip")
 
-# Code below can be used when tigris is updated to allow for 118th Congress.
-# congress_sf <- tigris::congressional_districts(year = BASE_YEAR) %>%
+# Set temp dir to download and extract ZIP files into
+congress_unzip_dir <- tempdir()
+
+# URL for current shapefile
+congress_zip_url <- c("https://www2.census.gov/geo/tiger/TIGER_RD18/LAYER/CD/tl_rd22_17_cd118.zip")
+
+# Create temporary file, download, and unzip to temp directory before unlinking
+congress_zip <- tempfile()
+download.file(congress_zip_url, congress_zip)
+unzip(congress_zip, exdir = congress_unzip_dir)
+unlink(congress_zip)
+
+#################################################################
 
-# The first line of this function should be replaced with the commented function
-# above when tigris is updated
+
+#### Commnted code below can be used when tigris is updated to allow for 118th
+#### Congress.
+
+#### The first line of this function should be replaced with the commented
+#### line below when tigris is updated
 ######################################
-congress_sf <- sf::read_sf("tl_rd22_17_cd118.shp") %>%
-  # The commented line below should be uncommented and replace the subsequent
-  # line when tigris is updated
+congress_sf <- sf::read_sf(paste(congress_unzip_dir,"tl_rd22_17_cd118.shp",sep="\\")) %>%
 ######################################
-  # filter(STATEFP == STATE, LSAD == "C2") %>%
+# congress_sf <- tigris::congressional_districts(year = BASE_YEAR) %>%
 ######################################
 
+
+#### The commented line below should be uncommented and replace the subsequent
+#### line when tigris is updated
+######################################
+  # filter(STATEFP == STATE, LSAD == "C2") %>%
+######################################
   filter(STATEFP20 == STATE, LSAD20 == "C2") %>%
+######################################
   sf::st_transform(cmap_crs) %>%
   mutate(dist_num = as.integer(CD118FP),
          dist_name = paste("Illinois",
@@ -105,18 +123,6 @@ congress_sf <- sf::read_sf("tl_rd22_17_cd118.shp") %>%
   select(dist_num, dist_name, dist_name_short, cmap, sqmi) %>%
   arrange(dist_num)
 
-# To remove when tigris is updated
-######################################
-file.remove("cd_2022.zip",
-            "tl_rd22_17_cd118.shx",
-            "tl_rd22_17_cd118.cpg",
-            "tl_rd22_17_cd118.dbf",
-            "tl_rd22_17_cd118.prj",
-            "tl_rd22_17_cd118.shp",
-            "tl_rd22_17_cd118.shp.ea.iso.xml",
-            "tl_rd22_17_cd118.shp.iso.xml")
-######################################
-
 
 # Process IL House Districts (Illinois General Assembly)
 ilga_house_sf <- tigris::state_legislative_districts(state = STATE, house = "lower", year = BASE_YEAR) %>%

diff --git a/data-raw/load_datahub_shp.R b/data-raw/load_datahub_shp.R
@@ -1,39 +1,38 @@
 library(tidyverse)
 devtools::load_all()
 
-# Note - this script is temporarily nonfunctional because CMAP's data hub is offline.
-
 # Links to zipped shapefiles hosted on the CMAP Data Hub
-CMAP_ZIP_URL <- "https://datahub.cmap.illinois.gov/dataset/2e8ed4ce-d056-4183-8d97-5f2901edb1f9/resource/832c29a6-9ca8-4459-9ff0-106c56c39a0d/download/mpocountiescmap201409.zip"
-COM_ZIP_URL <- "https://datahub.cmap.illinois.gov/dataset/a466a0bf-3c3e-48cb-8297-6eb55f49fb50/resource/cbe742e0-2e78-4a54-95c2-d694703e1ae6/download/CoMCMAP201303.zip"
-EDA_ZIP_URL <- "https://datahub.cmap.illinois.gov/dataset/21062031-c02b-4547-bbe3-6c45ba50ebc4/resource/e569ee54-00d4-430f-84b4-089ff3c00e53/download/DisinvestedandEDA.zip"
-FPA_ZIP_URL <- "https://datahub.cmap.illinois.gov/dataset/18c6875e-e0d1-4e97-aa4f-6a7b7d435d16/resource/df877dc8-2924-49ae-b76c-32b9c895f424/download/FPACMAP201404rev1.zip"
-SUBZONE_ZIP_URL <- "https://datahub.cmap.illinois.gov/dataset/a515b107-bdee-4b4c-b92e-50d3ecc0d971/resource/c4e47fca-0030-4b66-9221-947120c9c24f/download/subzones17CMAP2019.zip"
-ZONE_ZIP_URL <- "https://datahub.cmap.illinois.gov/dataset/a515b107-bdee-4b4c-b92e-50d3ecc0d971/resource/8dd37215-98dc-4244-9623-2d28c4e1611c/download/zones17.zip"
+
+### CMAP MPA file is now offline - will need to be updated with new source when posted
+# CMAP_ZIP_URL <- "https://datahub.cmap.illinois.gov/dataset/2e8ed4ce-d056-4183-8d97-5f2901edb1f9/resource/832c29a6-9ca8-4459-9ff0-106c56c39a0d/download/mpocountiescmap201409.zip"
+
+# Other files now use CMAP's new data hub structure.
+COM_URL <- "https://services5.arcgis.com/LcMXE3TFhi1BSaCY/arcgis/rest/services/Councils_of_Mayors_Boundary_Shapefile_March_2013/FeatureServer/0/query?where=1%3D1&outFields=*&outSR=4326&f=json"
+EDA_URL <- "https://services5.arcgis.com/LcMXE3TFhi1BSaCY/arcgis/rest/services/EDAs_and_Disinvested_Areas/FeatureServer/0/query?where=1%3D1&outFields=*&outSR=4326&f=json"
+FPA_URL <- "https://services5.arcgis.com/LcMXE3TFhi1BSaCY/arcgis/rest/services/FPA_CMAP_201404_revised_6_16_/FeatureServer/0/query?where=1%3D1&outFields=*&outSR=4326&f=json"
+SUBZONE_URL <- "https://services5.arcgis.com/LcMXE3TFhi1BSaCY/arcgis/rest/services/Trip_Generation_Zone_Subzone_Geography_2017/FeatureServer/0/query?where=1%3D1&outFields=*&outSR=4326&f=json"
+ZONE_URL <- "https://services5.arcgis.com/LcMXE3TFhi1BSaCY/arcgis/rest/services/Traffic_Analysis_Zone_Geography_2017/FeatureServer/0/query?where=1%3D1&outFields=*&outSR=4326&f=json"
 
 # Set temp dir to download and extract ZIP files into
-unzip_dir <- tempdir()
+# unzip_dir <- tempdir()
 
-# Process CMAP Metropolitan Planning Area
-cmap_zip <- tempfile()
-download.file(CMAP_ZIP_URL, cmap_zip)
-unzip(cmap_zip, exdir = unzip_dir)
-unlink(cmap_zip)
-cmap_mpa_sf <- sf::st_read(paste(unzip_dir, "MPOcounties_CMAP_201409.shp", sep="\\")) %>%
-  sf::st_transform(cmap_crs) %>%
-  rename(label_name = LabelName) %>%
-  mutate(county_fips = paste0("17", FIPSCNTY),
-         whole_county = !(CNTYNAME %in% c("DeKalb", "Grundy")),
-         sqmi = unclass(sf::st_area(geometry) / sqft_per_sqmi)) %>%
-  select(label_name, county_fips, whole_county, sqmi) %>%
-  arrange(desc(whole_county), county_fips)
+#### Process CMAP Metropolitan Planning Area (code temporarily not functional
+#### due to outdated URL)
+# cmap_zip <- tempfile()
+# download.file(CMAP_ZIP_URL, cmap_zip)
+# unzip(cmap_zip, exdir = unzip_dir)
+# unlink(cmap_zip)
+# cmap_mpa_sf <- sf::st_read(paste(unzip_dir, "MPOcounties_CMAP_201409.shp", sep="\\")) %>%
+#   sf::st_transform(cmap_crs) %>%
+#   rename(label_name = LabelName) %>%
+#   mutate(county_fips = paste0("17", FIPSCNTY),
+#          whole_county = !(CNTYNAME %in% c("DeKalb", "Grundy")),
+#          sqmi = unclass(sf::st_area(geometry) / sqft_per_sqmi)) %>%
+#   select(label_name, county_fips, whole_county, sqmi) %>%
+#   arrange(desc(whole_county), county_fips)
 
 # Process Councils of Mayors
-com_zip <- tempfile()
-download.file(COM_ZIP_URL, com_zip)
-unzip(com_zip, exdir = unzip_dir)
-unlink(com_zip)
-com_sf <- sf::st_read(paste(unzip_dir, "CoM_CMAP_201303.shp", sep="\\")) %>%
+com_sf <- sf::st_read(COM_URL) %>%
   sf::st_transform(cmap_crs) %>%
   rename(council = Council,
          full_name = FullName) %>%
@@ -42,11 +41,8 @@ com_sf <- sf::st_read(paste(unzip_dir, "CoM_CMAP_201303.shp", sep="\\")) %>%
   arrange(council)
 
 # Process EDAs and Disinvested Areas
-eda_zip <- tempfile()
-download.file(EDA_ZIP_URL, eda_zip)
-unzip(eda_zip, exdir = unzip_dir)
-unlink(eda_zip)
-eda_sf <- sf::st_read(paste(unzip_dir, "Disinvested_and_EDA.shp", sep="\\")) %>%
+eda_sf <- sf::st_read(EDA_URL) %>%
+  sf::st_transform(cmap_crs) %>%
   filter(DisplayGro %in% c("Economically Disconnected Area", "Disinvested", "Both")) %>%
   sf::st_transform(cmap_crs) %>%
   rename(geoid_tract = GEOID10) %>%
@@ -63,11 +59,7 @@ eda_sf <- sf::st_read(paste(unzip_dir, "Disinvested_and_EDA.shp", sep="\\")) %>%
   arrange(geoid_tract)
 
 # Process Facility Planning Areas
-fpa_zip <- tempfile()
-download.file(FPA_ZIP_URL, fpa_zip)
-unzip(fpa_zip, exdir = unzip_dir)
-unlink(fpa_zip)
-fpa_sf <- sf::st_read(paste(unzip_dir, "FPA_CMAP_201404rev1.shp", sep="\\")) %>%
+fpa_sf <- sf::st_read(FPA_URL) %>%
   filter(FPANAME != "Non-FPA") %>%
   sf::st_transform(cmap_crs) %>%
   rename(fpa = FPANAME,
@@ -79,11 +71,7 @@ fpa_sf <- sf::st_read(paste(unzip_dir, "FPA_CMAP_201404rev1.shp", sep="\\")) %>%
   arrange(parent_fpa, sub_fpa)
 
 # Process subzones
-subzone_zip <- tempfile()
-download.file(SUBZONE_ZIP_URL, subzone_zip)
-unzip(subzone_zip, exdir = unzip_dir)
-unlink(subzone_zip)
-subzone_sf <- sf::st_read(paste(unzip_dir, "subzones17.shp", sep="\\")) %>%
+subzone_sf <- sf::st_read(SUBZONE_URL) %>%
   sf::st_transform(cmap_crs) %>%
   rename(county_fips = county_fip,
          township_range = township_r,
@@ -99,11 +87,7 @@ subzone_sf <- sf::st_read(paste(unzip_dir, "subzones17.shp", sep="\\")) %>%
   arrange(subzone17)
 
 # Process zones
-zone_zip <- tempfile()
-download.file(ZONE_ZIP_URL, zone_zip)
-unzip(zone_zip, exdir = unzip_dir)
-unlink(zone_zip)
-zone_sf <- sf::st_read(paste(unzip_dir, "zones17.shp", sep="\\")) %>%
+zone_sf <- sf::st_read(ZONE_URL) %>%
   sf::st_transform(cmap_crs) %>%
   rename(county_fips = county_fip,
          township_range = township_r,

diff --git a/data/com_sf.rda b/data/com_sf.rda
diff --git a/data/eda_sf.rda b/data/eda_sf.rda
diff --git a/data/fpa_sf.rda b/data/fpa_sf.rda
diff --git a/data/subzone_sf.rda b/data/subzone_sf.rda
diff --git a/data/zone_sf.rda b/data/zone_sf.rda