add verbose argument to get_education_data to optionally quash wa…

…rnings (#109) * add verbose argument to data calls * adds test to verify that verbose arg quashes warnings * documentation update
UrbanInstitute · Jun 21, 2023 · 3aad05d · 3aad05d
1 parent 323c302
commit 3aad05d
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 19 deletions.
diff --git a/R/download-csv.R b/R/download-csv.R
@@ -86,8 +86,10 @@ get_csv_cols <- function(endpoints, urls, url_path) {
 # retrieve and read in a single csv file from a download url
 #
 # returns a data.frame for a single csv file
-download_csv <- function(url, cols, filters) {
-  message('\nFetching data for ', basename(url), ' ...')
+download_csv <- function(url, cols, filters, verbose = TRUE) {
+  if (verbose) {
+    message('\nFetching data for ', basename(url), ' ...')
+  }
   df <- readr::read_csv(url, col_types = cols, na = c('', '.'))
   df <- apply_csv_filters(df, filters)
   return(df)
@@ -96,8 +98,8 @@ download_csv <- function(url, cols, filters) {
 # retrieve and combine all csv files for an api endpoint
 #
 # returns a full data.frame for an api endpoint
-get_csv_data <- function(urls, cols, filters) {
-  dfs <- lapply(urls, function(x) download_csv(x, cols, filters))
+get_csv_data <- function(urls, cols, filters, verbose) {
+  dfs <- lapply(urls, function(x) download_csv(x, cols, filters, verbose))
   df <- do.call(rbind, dfs)
   return(df)
 }

diff --git a/R/get-page-data.R b/R/get-page-data.R
@@ -1,7 +1,7 @@
 # retrieve results from all pages of an api query for a single year
 #
 # returns data.frame, or an error if query fails
-get_year_data <- function(url) {
+get_year_data <- function(url, verbose = TRUE) {
 
   if (!grepl("stg", url)) {
     fetch <- gsub("https://educationdata.urban.org/api/v1/", "", url)
@@ -11,7 +11,9 @@ get_year_data <- function(url) {
   }
   fetch <- gsub('\\mode=R&', '', fetch)
   fetch <- gsub('\\mode=R', '', fetch)
-  message('\nFetching data for ', fetch, ' ...')
+  if (verbose) {
+    message('\nFetching data for ', fetch, ' ...')
+  }
   request <- httr::GET(url)
 
   if (request$status_code == 504) {
@@ -33,7 +35,9 @@ get_year_data <- function(url) {
   expected_rows <- resp$count
 
   if (expected_rows == 0) {
-    warning('Query ', url, ' returned no results.', call. = FALSE)
+    if (verbose) {
+      warning('Query ', url, ' returned no results.', call. = FALSE)
+    }
     df <- data.frame()
     return(df)
   }
@@ -47,7 +51,9 @@ get_year_data <- function(url) {
 
   while (!(is.null(url))) {
     count = count + 1
-    message(paste("Processing page", count, 'out of', pages))
+    if (verbose) {
+      message(paste("Processing page", count, 'out of', pages))
+    }
     request <- httr::GET(url)
 
     if (request$status_code == 504) {
@@ -72,7 +78,7 @@ get_year_data <- function(url) {
 
   df <- do.call(rbind, dfs)
 
-  if (nrow(df) != expected_rows) {
+  if (nrow(df) != expected_rows & verbose) {
     warning('API call expected ', expected_rows, ' results but received ',
             nrow(df), '. Consider filing an issue with the development team.',
             call. = FALSE)
@@ -84,8 +90,8 @@ get_year_data <- function(url) {
 # retrieve results from all pages of an api query across all given years
 #
 # returns data.frame
-get_all_data <- function(urls) {
-  dfs <- lapply(urls, get_year_data)
+get_all_data <- function(urls, verbose = TRUE) {
+  dfs <- lapply(urls, function(x) get_year_data(x, verbose))
   df <- do.call(rbind, dfs)
   return(df)
 }
diff --git a/R/get_education_data.R b/R/get_education_data.R
@@ -8,6 +8,7 @@
 #' @param filters Optional 'list' of query values to filter an API call
 #' @param add_labels Add variable labels (when applicable)? Defaults to FALSE.
 #' @param csv Download the full csv file? Defaults to FALSE.
+#' @param verbose Print messages and warnings?  Defaults to TRUE
 #'
 #' @return A `data.frame` of education data
 #'
@@ -19,7 +20,8 @@ get_education_data <- function(level = NULL,
                                by = NULL,
                                filters = NULL,
                                add_labels = FALSE,
-                               csv = FALSE) {
+                               csv = FALSE,
+                               verbose = TRUE) {
 
   if (!is.null(by)) {
     warning("The `by` argument has been deprecated in favor of `subtopic`.\n",
@@ -30,9 +32,13 @@ get_education_data <- function(level = NULL,
 
 
   if (csv) {
-    df <- get_education_data_csv(level, source, topic, subtopic, filters, add_labels)
+    df <- get_education_data_csv(
+      level, source, topic, subtopic, filters, add_labels, verbose
+    )
   } else {
-    df <- get_education_data_json(level, source, topic, subtopic, filters, add_labels)
+    df <- get_education_data_json(
+      level, source, topic, subtopic, filters, add_labels, verbose
+    )
   }
 
   return(df)
@@ -48,7 +54,8 @@ get_education_data_json <- function(level = NULL,
                                     topic = NULL,
                                     by = NULL,
                                     filters = NULL,
-                                    add_labels = FALSE) {
+                                    add_labels = FALSE,
+                                    verbose = TRUE) {
 
   url_path <- "https://educationdata.urban.org"
 
@@ -65,7 +72,7 @@ get_education_data_json <- function(level = NULL,
                         filters  = filters,
                         url_path)
 
-  df <- get_all_data(urls)
+  df <- get_all_data(urls, verbose)
 
   if(add_labels & nrow(df) != 0) {
     df <- add_variable_labels(endpoints, df, url_path)
@@ -82,7 +89,8 @@ get_education_data_csv <- function(level = NULL,
                                    topic = NULL,
                                    by = NULL,
                                    filters = NULL,
-                                   add_labels = FALSE) {
+                                   add_labels = FALSE,
+                                   verbose = TRUE) {
 
   url_path <- "https://educationdata.urban.org"
 
@@ -105,7 +113,7 @@ get_education_data_csv <- function(level = NULL,
 
   cols <- get_csv_cols(endpoints, urls, url_path)
 
-  df <- get_csv_data(urls, cols, filters)
+  df <- get_csv_data(urls, cols, filters, verbose)
 
   if(add_labels & nrow(df) != 0) {
     df <- add_variable_labels(endpoints, df, url_path)

diff --git a/man/get_education_data.Rd b/man/get_education_data.Rd
diff --git a/tests/testthat/test-validation.R b/tests/testthat/test-validation.R
@@ -41,3 +41,18 @@ test_that('invalid api arguments return an error message', {
                                                    fips = 1),
                                     by = list("race", "sex")))
 })
+
+
+test_that('can quiet api messages with verbose = FALSE', {
+  skip_on_cran()
+
+  expect_warning(
+    get_education_data(level = 'schools',
+                       source = 'ccd',
+                       topic = 'directory',
+                       filters = list(year = 2014,
+                                      fips = 100),
+                       verbose = FALSE),
+    regexp = NA
+  )
+})