diff --git a/Gemfile b/Gemfile index 4f574a1..625ded9 100644 --- a/Gemfile +++ b/Gemfile @@ -32,3 +32,5 @@ gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] # Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem # do not have a Java counterpart. gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby] + +gem "webrick", "~> 1.8" diff --git a/Gemfile.lock b/Gemfile.lock index 32dd189..901d4d2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -212,6 +212,8 @@ GEM minitest (5.20.0) nokogiri (1.16.2-arm64-darwin) racc (~> 1.4) + nokogiri (1.16.2-x64-mingw-ucrt) + racc (~> 1.4) nokogiri (1.16.2-x86_64-linux) racc (~> 1.4) octokit (4.25.1) @@ -249,6 +251,7 @@ GEM unf_ext unf_ext (0.0.8.2) unicode-display_width (1.8.0) + webrick (1.8.1) PLATFORMS arm64-darwin-21 @@ -262,6 +265,7 @@ DEPENDENCIES tzinfo (>= 1, < 3) tzinfo-data wdm (~> 0.1.1) + webrick (~> 1.8) BUNDLED WITH 2.3.16 diff --git a/catalogs/build-catalog-functions.R b/catalogs/build-catalog-functions.R index 7b3c21a..3373f6e 100644 --- a/catalogs/build-catalog-functions.R +++ b/catalogs/build-catalog-functions.R @@ -138,7 +138,7 @@ get_file_paths <- function(series, } else if (series == "core"){ expr <- paste0( "CORE-[0-9]{4}-501C[0-9A-Z]-", tscope ) paths <- grep( expr, paths, value = TRUE ) - paths <- grep( paste0( "-", fscope, "\\b"), paths, value=T ) + paths <- grep( paste0( "-", fscope, "\\.csv"), paths, value=T ) } else if (series == "misc"){ expr <- "SUPPLEMENTAL-CORE.*" paths <- grep(expr, paths, value = TRUE) @@ -200,8 +200,12 @@ make_archive_urls <- function(series, matches <- gsub("\\.csv", "", matches) archive_urls <- paste0(base_url, matches) - # archive_urls <- lapply(archive_urls, - # function(x) if (RCurl::url.exists(x)) x else unavail_url) + archive_urls <- lapply(archive_urls, + function(x) + if (RCurl::url.exists(x)) + x + else + unavail_url) return(archive_urls) } diff --git a/catalogs/catalog-core.html b/catalogs/catalog-core.html index 5bb3bf8..6751d75 100644 --- a/catalogs/catalog-core.html +++ b/catalogs/catalog-core.html @@ -306,7 +306,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 343.2 mb 2014 501C3-CHARITIES @@ -314,7 +314,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 348.3 mb 2015 501C3-CHARITIES @@ -322,7 +322,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 339.1 mb 2016 501C3-CHARITIES @@ -330,7 +330,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 349.4 mb 2017 501C3-CHARITIES @@ -338,7 +338,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 351.8 mb 2018 501C3-CHARITIES @@ -346,7 +346,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 355.1 mb 2019 501C3-CHARITIES @@ -395,7 +395,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 228.4 mb 2014 501C3-CHARITIES @@ -403,7 +403,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 240.5 mb 2015 501C3-CHARITIES @@ -411,7 +411,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 236.6 mb 2016 501C3-CHARITIES @@ -419,7 +419,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 242.6 mb 2017 501C3-CHARITIES @@ -427,7 +427,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 241.5 mb 2018 501C3-CHARITIES @@ -435,7 +435,7 @@

501c3 CHARITIES SCOP DOWNLOAD -PROFILE +PROFILE 223.2 mb 2019 501C3-CHARITIES @@ -629,7 +629,7 @@

NONPROFITS SCOPE PZ DOWNLOAD -PROFILE +PROFILE 96.1 mb 2009 501CE-CHARITIES @@ -637,7 +637,7 @@

NONPROFITS SCOPE PZ DOWNLOAD -PROFILE +PROFILE 98 mb 2010 501CE-CHARITIES @@ -669,7 +669,7 @@

NONPROFITS SCOPE PZ DOWNLOAD -PROFILE +PROFILE 115.7 mb 2014 501CE-CHARITIES @@ -677,7 +677,7 @@

NONPROFITS SCOPE PZ DOWNLOAD -PROFILE +PROFILE 115 mb 2015 501CE-CHARITIES @@ -685,7 +685,7 @@

NONPROFITS SCOPE PZ DOWNLOAD -PROFILE +PROFILE 113.6 mb 2016 501CE-CHARITIES @@ -693,7 +693,7 @@

NONPROFITS SCOPE PZ DOWNLOAD -PROFILE +PROFILE 110.8 mb 2017 501CE-CHARITIES @@ -701,7 +701,7 @@

NONPROFITS SCOPE PZ DOWNLOAD -PROFILE +PROFILE 111.1 mb 2018 501CE-CHARITIES @@ -709,7 +709,7 @@

NONPROFITS SCOPE PZ DOWNLOAD -PROFILE +PROFILE 110.7 mb 2019 501CE-CHARITIES @@ -759,7 +759,7 @@

NONPROFITS SCOPE PC DOWNLOAD -PROFILE +PROFILE 76.3 mb 2014 501CE-NONPROFIT @@ -767,7 +767,7 @@

NONPROFITS SCOPE PC DOWNLOAD -PROFILE +PROFILE 79.3 mb 2015 501CE-NONPROFIT @@ -775,7 +775,7 @@

NONPROFITS SCOPE PC DOWNLOAD -PROFILE +PROFILE 76.9 mb 2016 501CE-NONPROFIT @@ -783,7 +783,7 @@

NONPROFITS SCOPE PC DOWNLOAD -PROFILE +PROFILE 77.4 mb 2017 501CE-NONPROFIT @@ -791,7 +791,7 @@

NONPROFITS SCOPE PC DOWNLOAD -PROFILE +PROFILE 75.6 mb 2018 501CE-NONPROFIT @@ -799,7 +799,7 @@

NONPROFITS SCOPE PC DOWNLOAD -PROFILE +PROFILE 64.4 mb 2019 501CE-NONPROFIT @@ -1025,7 +1025,7 @@

PRIVATE FOUND DOWNLOAD -PROFILE +PROFILE 72.5 mb 2014 501C3-CHARITIES @@ -1033,7 +1033,7 @@

PRIVATE FOUND DOWNLOAD -PROFILE +PROFILE 73.9 mb 2015 501C3-CHARITIES @@ -1041,7 +1041,7 @@

PRIVATE FOUND DOWNLOAD -PROFILE +PROFILE 57.4 mb 2019 501C3-CHARITIES diff --git a/catalogs/catalog-core.qmd b/catalogs/catalog-core.qmd index e44e1f7..385ed00 100644 --- a/catalogs/catalog-core.qmd +++ b/catalogs/catalog-core.qmd @@ -18,7 +18,7 @@ library( pander ) GH.RAW <- "https://raw.githubusercontent.com/UrbanInstitute/nccs/main/catalogs/" d <- read.csv( paste0( GH.RAW, "AWS-NCCSDATA.csv" ) ) -source( paste0( GH.RAW, "build-catalog-functions.R" ) ) +source("build-catalog-functions.R") nccs_legacy_base_url <- "https://urbaninstitute.github.io/nccs-legacy/dictionary/core/core_archive_html/" ```