From 3aad05d79c08e20f54bd5761595b68c37e52f388 Mon Sep 17 00:00:00 2001 From: Andrew Martin Date: Wed, 21 Jun 2023 13:04:09 -0400 Subject: [PATCH] add `verbose` argument to `get_education_data` to optionally quash warnings (#109) * add verbose argument to data calls * adds test to verify that verbose arg quashes warnings * documentation update --- R/download-csv.R | 10 ++++++---- R/get-page-data.R | 20 +++++++++++++------- R/get_education_data.R | 22 +++++++++++++++------- man/get_education_data.Rd | 5 ++++- tests/testthat/test-validation.R | 15 +++++++++++++++ 5 files changed, 53 insertions(+), 19 deletions(-) diff --git a/R/download-csv.R b/R/download-csv.R index 56efaee..c182176 100644 --- a/R/download-csv.R +++ b/R/download-csv.R @@ -86,8 +86,10 @@ get_csv_cols <- function(endpoints, urls, url_path) { # retrieve and read in a single csv file from a download url # # returns a data.frame for a single csv file -download_csv <- function(url, cols, filters) { - message('\nFetching data for ', basename(url), ' ...') +download_csv <- function(url, cols, filters, verbose = TRUE) { + if (verbose) { + message('\nFetching data for ', basename(url), ' ...') + } df <- readr::read_csv(url, col_types = cols, na = c('', '.')) df <- apply_csv_filters(df, filters) return(df) @@ -96,8 +98,8 @@ download_csv <- function(url, cols, filters) { # retrieve and combine all csv files for an api endpoint # # returns a full data.frame for an api endpoint -get_csv_data <- function(urls, cols, filters) { - dfs <- lapply(urls, function(x) download_csv(x, cols, filters)) +get_csv_data <- function(urls, cols, filters, verbose) { + dfs <- lapply(urls, function(x) download_csv(x, cols, filters, verbose)) df <- do.call(rbind, dfs) return(df) } diff --git a/R/get-page-data.R b/R/get-page-data.R index 0f1ccd9..c5ca775 100644 --- a/R/get-page-data.R +++ b/R/get-page-data.R @@ -1,7 +1,7 @@ # retrieve results from all pages of an api query for a single year # # returns data.frame, or an error if query fails -get_year_data <- function(url) { +get_year_data <- function(url, verbose = TRUE) { if (!grepl("stg", url)) { fetch <- gsub("https://educationdata.urban.org/api/v1/", "", url) @@ -11,7 +11,9 @@ get_year_data <- function(url) { } fetch <- gsub('\\mode=R&', '', fetch) fetch <- gsub('\\mode=R', '', fetch) - message('\nFetching data for ', fetch, ' ...') + if (verbose) { + message('\nFetching data for ', fetch, ' ...') + } request <- httr::GET(url) if (request$status_code == 504) { @@ -33,7 +35,9 @@ get_year_data <- function(url) { expected_rows <- resp$count if (expected_rows == 0) { - warning('Query ', url, ' returned no results.', call. = FALSE) + if (verbose) { + warning('Query ', url, ' returned no results.', call. = FALSE) + } df <- data.frame() return(df) } @@ -47,7 +51,9 @@ get_year_data <- function(url) { while (!(is.null(url))) { count = count + 1 - message(paste("Processing page", count, 'out of', pages)) + if (verbose) { + message(paste("Processing page", count, 'out of', pages)) + } request <- httr::GET(url) if (request$status_code == 504) { @@ -72,7 +78,7 @@ get_year_data <- function(url) { df <- do.call(rbind, dfs) - if (nrow(df) != expected_rows) { + if (nrow(df) != expected_rows & verbose) { warning('API call expected ', expected_rows, ' results but received ', nrow(df), '. Consider filing an issue with the development team.', call. = FALSE) @@ -84,8 +90,8 @@ get_year_data <- function(url) { # retrieve results from all pages of an api query across all given years # # returns data.frame -get_all_data <- function(urls) { - dfs <- lapply(urls, get_year_data) +get_all_data <- function(urls, verbose = TRUE) { + dfs <- lapply(urls, function(x) get_year_data(x, verbose)) df <- do.call(rbind, dfs) return(df) } diff --git a/R/get_education_data.R b/R/get_education_data.R index 499ebc2..0aefe64 100644 --- a/R/get_education_data.R +++ b/R/get_education_data.R @@ -8,6 +8,7 @@ #' @param filters Optional 'list' of query values to filter an API call #' @param add_labels Add variable labels (when applicable)? Defaults to FALSE. #' @param csv Download the full csv file? Defaults to FALSE. +#' @param verbose Print messages and warnings? Defaults to TRUE #' #' @return A `data.frame` of education data #' @@ -19,7 +20,8 @@ get_education_data <- function(level = NULL, by = NULL, filters = NULL, add_labels = FALSE, - csv = FALSE) { + csv = FALSE, + verbose = TRUE) { if (!is.null(by)) { warning("The `by` argument has been deprecated in favor of `subtopic`.\n", @@ -30,9 +32,13 @@ get_education_data <- function(level = NULL, if (csv) { - df <- get_education_data_csv(level, source, topic, subtopic, filters, add_labels) + df <- get_education_data_csv( + level, source, topic, subtopic, filters, add_labels, verbose + ) } else { - df <- get_education_data_json(level, source, topic, subtopic, filters, add_labels) + df <- get_education_data_json( + level, source, topic, subtopic, filters, add_labels, verbose + ) } return(df) @@ -48,7 +54,8 @@ get_education_data_json <- function(level = NULL, topic = NULL, by = NULL, filters = NULL, - add_labels = FALSE) { + add_labels = FALSE, + verbose = TRUE) { url_path <- "https://educationdata.urban.org" @@ -65,7 +72,7 @@ get_education_data_json <- function(level = NULL, filters = filters, url_path) - df <- get_all_data(urls) + df <- get_all_data(urls, verbose) if(add_labels & nrow(df) != 0) { df <- add_variable_labels(endpoints, df, url_path) @@ -82,7 +89,8 @@ get_education_data_csv <- function(level = NULL, topic = NULL, by = NULL, filters = NULL, - add_labels = FALSE) { + add_labels = FALSE, + verbose = TRUE) { url_path <- "https://educationdata.urban.org" @@ -105,7 +113,7 @@ get_education_data_csv <- function(level = NULL, cols <- get_csv_cols(endpoints, urls, url_path) - df <- get_csv_data(urls, cols, filters) + df <- get_csv_data(urls, cols, filters, verbose) if(add_labels & nrow(df) != 0) { df <- add_variable_labels(endpoints, df, url_path) diff --git a/man/get_education_data.Rd b/man/get_education_data.Rd index 15aa665..6444647 100644 --- a/man/get_education_data.Rd +++ b/man/get_education_data.Rd @@ -12,7 +12,8 @@ get_education_data( by = NULL, filters = NULL, add_labels = FALSE, - csv = FALSE + csv = FALSE, + verbose = TRUE ) } \arguments{ @@ -31,6 +32,8 @@ get_education_data( \item{add_labels}{Add variable labels (when applicable)? Defaults to FALSE.} \item{csv}{Download the full csv file? Defaults to FALSE.} + +\item{verbose}{Print messages and warnings? Defaults to TRUE} } \value{ A `data.frame` of education data diff --git a/tests/testthat/test-validation.R b/tests/testthat/test-validation.R index 09fa60e..e314310 100644 --- a/tests/testthat/test-validation.R +++ b/tests/testthat/test-validation.R @@ -41,3 +41,18 @@ test_that('invalid api arguments return an error message', { fips = 1), by = list("race", "sex"))) }) + + +test_that('can quiet api messages with verbose = FALSE', { + skip_on_cran() + + expect_warning( + get_education_data(level = 'schools', + source = 'ccd', + topic = 'directory', + filters = list(year = 2014, + fips = 100), + verbose = FALSE), + regexp = NA + ) +})