diff --git a/DESCRIPTION b/DESCRIPTION index d302e5b..376e3c3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,12 +14,14 @@ Imports: cachem, docopt, fs, + ggplot2, httpuv, jsonlite, logger, lubridate, mgcv, mime, + plotly, plumber, porcelain, rlang, diff --git a/R/api.R b/R/api.R index 13b67a4..d4cb828 100644 --- a/R/api.R +++ b/R/api.R @@ -90,9 +90,7 @@ target_get_dataset <- function(name, req) { variables <- list() for (col in cols) { lvls <- unique(dat[, col]) - if (length(lvls) < 25) { - variables[[col]] <- list(name = jsonlite::unbox(col), levels = lvls) - } + variables[[col]] <- list(name = jsonlite::unbox(col), levels = lvls) } list(variables = unname(variables), biomarkers = biomarkers, @@ -113,22 +111,14 @@ target_get_trace <- function(name, method = "auto", span = 0.75, k = 10) { - biomarker <- httpuv::decodeURIComponent(biomarker) + biomarker <- httpuv::decodeURIComponent(biomarker) logger::log_info(paste("Requesting data from", name, "with biomarker", biomarker)) dataset <- read_dataset(req, name, scale) dat <- dataset$data xcol <- dataset$xcol xtype <- dataset$xtype - cols <- colnames(dat) - if (!is.null(filter)) { - filters <- strsplit(filter, "+", fixed = TRUE)[[1]] - logger::log_info(paste("Filtering by variables:", paste(filters, - collapse = ", "))) - for (f in filters) { - dat <- apply_filter(f, dat, cols) - } - } + dat <- apply_filters(dat, filter) dat <- dat[dat["biomarker"] == biomarker, ] if (length(disaggregate) > 0) { logger::log_info(paste("Disaggregating by variables:", disaggregate)) @@ -162,6 +152,73 @@ target_get_trace <- function(name, } } +target_get_individual <- function(req, + name, + pidcol, + scale = "natural", + filter = NULL, + color = NULL, + linetype = NULL, + page = 1) { + .data <- value <- NULL + + data <- read_dataset(req, name, scale) + dat <- data$data + xcol <- data$xcol + + if (!(pidcol %in% colnames(dat))) { + porcelain::porcelain_stop(sprintf("Id column '%s' not found.", pidcol)) + } + + dat <- apply_filters(dat, filter) + if (is.null(color)) { + if (is.null(linetype)) { + aes <- ggplot2::aes(x = .data[[xcol]], y = value) + } else { + aes <- ggplot2::aes(x = .data[[xcol]], y = value, + linetype = .data[[linetype]]) + } + } else { + if (is.null(linetype)) { + aes <- ggplot2::aes(x = .data[[xcol]], y = value, + color = .data[[color]]) + } else { + aes <- ggplot2::aes(x = .data[[xcol]], y = value, + color = .data[[color]], + linetype = .data[[linetype]]) + } + } + + warnings <- NULL + ids <- unique(dat[[pidcol]]) + if (length(ids) > 20) { + msg <- paste(length(ids), + "individuals identified; only the first 20 will be shown.") + warnings <- c(warnings, msg) + dat <- dat[dat[[pidcol]] %in% ids[1:20], ] + } + + # Facets in plotlyjs are quite a pain. Using ggplot2 and plotly R + # packages to generate the plotly data and layout objects is a bit slower + # than just generating data series in R and letting the front-end handle the + # presentation logic, but is much easier to get right! + p <- with_warnings(ggplot2::ggplot(dat) + + ggplot2::geom_line(aes) + + ggplot2::facet_wrap(pidcol) + + ggplot2::theme_bw() + + ggplot2::labs(x = xcol, y = "Antibody titre", + linetype = linetype, + color = color)) + warnings <- c(warnings, p$warnings) + + q <- plotly::ggplotly(p$output) + jsonlite::toJSON( + list(data = as.list(q$x$data), + layout = as.list(q$x$layout), + warnings = warnings), + auto_unbox = TRUE, null = "null") +} + read_dataset <- function(req, name, scale) { validate_scale(scale) session_id <- get_or_create_session_id(req) @@ -188,10 +245,10 @@ read_dataset <- function(req, name, scale) { } model_out <- function(dat, xcol, - xtype = "number", - method = "auto", - span = 0.75, - k = 10) { + xtype = "number", + method = "auto", + span = 0.75, + k = 10) { n <- nrow(dat) if (n == 0) { return(list(x = list(), y = list())) @@ -219,6 +276,19 @@ data_out <- function(dat, xcol) { list(x = dat[, xcol], y = dat$value) } +apply_filters <- function(dat, filter) { + if (!is.null(filter)) { + filters <- strsplit(filter, "+", fixed = TRUE)[[1]] + logger::log_info(paste("Filtering by variables:", paste(filters, + collapse = ", "))) + cols <- colnames(dat) + for (f in filters) { + dat <- apply_filter(f, dat, cols) + } + } + return(dat) +} + apply_filter <- function(filter, dat, cols) { filter_def <- strsplit(filter, ":") filter_var <- filter_def[[1]][1] diff --git a/R/router.R b/R/router.R index 0e9f86b..f8e743b 100644 --- a/R/router.R +++ b/R/router.R @@ -38,6 +38,7 @@ build_routes <- function(cookie_key = plumber::random_cookie_key(), pr$handle(get_dataset()) pr$handle(get_datasets()) pr$handle(get_trace()) + pr$handle(get_individual()) } get_root <- function() { @@ -85,6 +86,18 @@ get_trace <- function() { returning = porcelain::porcelain_returning_json("DataSeries")) } +get_individual <- function() { + porcelain::porcelain_endpoint$new( + "GET", + "/dataset//individual//", + target_get_individual, + porcelain::porcelain_input_query(scale = "string", + color = "string", + filter = "string", + linetype = "string"), + returning = porcelain::porcelain_returning_json()) +} + prune_inactive_sessions <- function(cache) { active_sessions <- cache$keys() subdirectories <- list.files("uploads") diff --git a/docker/Dockerfile b/docker/Dockerfile index 883dd43..097c1c6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,10 +16,12 @@ RUN install_packages --repo=https://mrc-ide.r-universe.dev \ callr \ docopt \ fs \ + ggplot2 \ logger \ lubridate \ jsonlite \ jsonvalidate \ + plotly \ plumber \ remotes \ Rook \ diff --git a/tests/testthat/test-read-individuals.R b/tests/testthat/test-read-individuals.R new file mode 100644 index 0000000..1e851f8 --- /dev/null +++ b/tests/testthat/test-read-individuals.R @@ -0,0 +1,236 @@ +test_that("GET /individual/ returns 404 if dataset not found", { + router <- build_routes(cookie_key) + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 404) + validate_failure_schema(res$body) + body <- jsonlite::fromJSON(res$body) + expect_equal(body$errors[1, "detail"], + "Did not find dataset with name: testdataset") +}) + +test_that("GET /individual/ returns 400 if pidcol not found", { + dat <- data.frame(biomarker = "ab", + value = 1, + day = 1:10) + local_add_dataset(dat, name = "testdataset") + router <- build_routes(cookie_key) + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 400) + validate_failure_schema(res$body) + body <- jsonlite::fromJSON(res$body) + str(body) + expect_equal(body$errors[1, "detail"], + "Id column 'pid' not found.") +}) + +test_that("can get individual trajectories for uploaded dataset with xcol", { + dat <- data.frame(biomarker = c("ab", "ba"), + pid = c(1, 2), + value = 11:20, + time = 1:10, + age = "0-5", + sex = c("M", "F")) + router <- build_routes(cookie_key) + post_request <- local_POST_dataset_request(dat, + "testdataset", + xcol = "time", + cookie = cookie) + expect_equal(router$call(post_request)$status, 200) + + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 200) + body <- jsonlite::fromJSON(res$body) + expect_equal(nrow(body$data$data), 2) + expect_equal(body$data$data$x[[1]], c(1, 3, 5, 7, 9)) + expect_equal(body$data$data$y[[1]], c(11, 13, 15, 17, 19)) + expect_equal(body$data$data$x[[2]], c(2, 4, 6, 8, 10)) + expect_equal(body$data$data$y[[2]], c(12, 14, 16, 18, 20)) +}) + +test_that("GET /individual/?scale= returns 400 if invalid scale", { + dat <- data.frame(biomarker = "ab", + value = 1, + day = 1:10, + age = "0-5", + sex = c("M", "F")) + local_add_dataset(dat, name = "testdataset") + router <- build_routes(cookie_key) + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + qs = "scale=bad", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 400) + validate_failure_schema(res$body) + body <- jsonlite::fromJSON(res$body) + expect_equal(body$errors[1, "detail"], + "'scale' must be one of 'log', 'log2', or 'natural'") +}) + +test_that("can get trajectories with color", { + dat <- data.frame(pid = c(1, 2), + value = 1, + day = 1:10, + age = "0-5", + sex = c("M", "F")) + local_add_dataset(dat, name = "testdataset") + router <- build_routes(cookie_key) + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + qs = "color=sex", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 200) + body <- jsonlite::fromJSON(res$body) + data <- body$data$data + expect_equal(nrow(data), 2) + expect_equal(data$name, c("F", "M")) + expect_false(data[1, "line"]$color == data[2, "line"]$color) + + expect_equal(data[1, "x"], list(c(2, 4, 6, 8, 10))) + expect_equal(data[1, "y"], list(c(1, 1, 1, 1, 1))) + expect_equal(data[1, "line"]$dash, "solid") + + + expect_equal(data[2, "x"], list(c(1, 3, 5, 7, 9))) + expect_equal(data[2, "y"], list(c(1, 1, 1, 1, 1))) + expect_equal(data[2, "line"]$dash, "solid") +}) + +test_that("can get trajectoriees with linetype", { + dat <- data.frame(pid = 1, + value = c(1, 2), + day = 1:10, + subtype = c("M", "F")) + router <- build_routes(cookie_key) + local_add_dataset(dat, name = "testdataset") + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + qs = "linetype=subtype", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 200) + body <- jsonlite::fromJSON(res$body) + data <- body$data$data + expect_equal(nrow(data), 2) + expect_equal(data$name, c("F", "M")) + expect_equal(data[1, "line"]$color, data[2, "line"]$color) + + expect_equal(data[1, "x"], list(c(2, 4, 6, 8, 10))) + expect_equal(data[1, "y"], list(rep(2, 5))) + expect_equal(data[1, "line"]$dash, "solid") + + expect_equal(data[2, "x"], list(c(1, 3, 5, 7, 9))) + expect_equal(data[2, "y"], list(rep(1, 5))) + expect_equal(data[2, "line"]$dash, "dash") +}) + +test_that("can get trajectoriees with linetype and color", { + dat <- data.frame(pid = 1, + value = c(1, 2), + day = 1:10, + biomarker = rep(c("ab", "ba"), 5), + subtype = c("M", "F")) + router <- build_routes(cookie_key) + local_add_dataset(dat, name = "testdataset") + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + qs = "linetype=subtype&color=biomarker", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 200) + body <- jsonlite::fromJSON(res$body) + data <- body$data$data + expect_equal(nrow(data), 2) + expect_equal(data$name, c("(ab,M)", "(ba,F)")) + expect_false(data[1, "line"]$color == data[2, "line"]$color) + expect_equal(data[1, "line"]$dash, "dash") + expect_equal(data[2, "line"]$dash, "solid") +}) + +test_that("can get filtered individual trajectories", { + dat <- data.frame(pid = 1, + value = 1:10, + day = 1:10, + sex = c("M", "F")) + router <- build_routes(cookie_key) + local_add_dataset(dat, name = "testdataset") + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + qs = "filter=sex%3AM", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 200) + body <- jsonlite::fromJSON(res$body) + data <- body$data$data + expect_equal(nrow(data), 1) + expect_equal(data[1, "x"], list(c(1, 3, 5, 7, 9))) + expect_equal(data[1, "y"], list(c(1, 3, 5, 7, 9))) +}) + +test_that("can get log data", { + dat <- data.frame(biomarker = "ab", + pid = 1, + value = 1:5, + day = 1:5) + router <- build_routes(cookie_key) + local_add_dataset(dat, name = "testdataset") + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + qs = "scale=log", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 200) + body <- jsonlite::fromJSON(res$body) + data <- body$data$data + expect_equal(nrow(data), 1) + expect_equal(data[1, "x"], list(1:5)) + expect_equal(unlist(data[1, "y"]), + jsonlite::fromJSON( + jsonlite::toJSON(log(1:5)) # convert to/from json for consistent rounding + )) +}) + +test_that("can get dataset with dates", { + dates <- c("2023/15/01", "2023/16/01", "2023/17/01", "2023/18/01", "2023/20/01") + dat <- data.frame(biomarker = "ab", + pid = 1, + value = 1:5, + day = dates) + router <- build_routes(cookie_key) + post_request <- local_POST_dataset_request(dat, + "testdataset", + cookie = cookie) + expect_equal(router$call(post_request)$status, 200) + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 200) + body <- jsonlite::fromJSON(res$body) + data <- body$data$data + expect_equal(nrow(data), 1) + expect_equal(unlist(data[1, "x"]), as.numeric(lubridate::ydm(dates))) + expect_equal(unlist(data[1, "y"]), 1:5) +}) + +test_that("only first 20 individuals are returned", { + dat <- data.frame(biomarker = "ab", + pid = 1:25, + value = 1, + day = 1:25) + router <- build_routes(cookie_key) + post_request <- local_POST_dataset_request(dat, + "testdataset", + cookie = cookie) + expect_equal(router$call(post_request)$status, 200) + res <- router$call(make_req("GET", + "/dataset/testdataset/individual/pid/", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 200) + body <- jsonlite::fromJSON(res$body) + warnings <- body$data$warnings + expect_equal(warnings, "25 individuals identified; only the first 20 will be shown.") + + data <- body$data$data + expect_equal(nrow(data), 20) +}) diff --git a/tests/testthat/test-read-metadata.R b/tests/testthat/test-read-metadata.R new file mode 100644 index 0000000..91c822e --- /dev/null +++ b/tests/testthat/test-read-metadata.R @@ -0,0 +1,22 @@ +test_that("GET /dataset returns 404 if no session cookie present", { + router <- build_routes(cookie_key) + res <- router$call(make_req("GET", + "/dataset/testdataset/")) + expect_equal(res$status, 404) + validate_failure_schema(res$body) + body <- jsonlite::fromJSON(res$body) + expect_equal(body$errors[1, "detail"], + "Did not find dataset with name: testdataset") +}) + +test_that("GET /dataset returns 404 if dataset not found", { + router <- build_routes(cookie_key) + res <- router$call(make_req("GET", + "/dataset/testdataset/", + HTTP_COOKIE = cookie)) + expect_equal(res$status, 404) + validate_failure_schema(res$body) + body <- jsonlite::fromJSON(res$body) + expect_equal(body$errors[1, "detail"], + "Did not find dataset with name: testdataset") +}) diff --git a/tests/testthat/test-read.R b/tests/testthat/test-read-trace.R similarity index 93% rename from tests/testthat/test-read.R rename to tests/testthat/test-read-trace.R index 1638003..100d1bb 100644 --- a/tests/testthat/test-read.R +++ b/tests/testthat/test-read-trace.R @@ -1,26 +1,3 @@ -test_that("GET /dataset returns 404 if no session cookie present", { - router <- build_routes(cookie_key) - res <- router$call(make_req("GET", - "/dataset/testdataset/")) - expect_equal(res$status, 404) - validate_failure_schema(res$body) - body <- jsonlite::fromJSON(res$body) - expect_equal(body$errors[1, "detail"], - "Did not find dataset with name: testdataset") -}) - -test_that("GET /dataset returns 404 if dataset not found", { - router <- build_routes(cookie_key) - res <- router$call(make_req("GET", - "/dataset/testdataset/", - HTTP_COOKIE = cookie)) - expect_equal(res$status, 404) - validate_failure_schema(res$body) - body <- jsonlite::fromJSON(res$body) - expect_equal(body$errors[1, "detail"], - "Did not find dataset with name: testdataset") -}) - test_that("GET /trace/ returns 404 if dataset not found", { router <- build_routes(cookie_key) res <- router$call(make_req("GET",