From 78930766f3fef2c8363841268796e66ae238a472 Mon Sep 17 00:00:00 2001 From: Samuel Brand Date: Fri, 4 Oct 2024 12:43:40 +0100 Subject: [PATCH 1/2] fix tests and capture forecasting failures --- pipeline/src/infer/InferenceConfig.jl | 2 +- pipeline/test/pipeline/test_pipeline.jl | 2 ++ .../test/pipeline/test_pipelinefunctions.jl | 36 +++++-------------- pipeline/test/runtests.jl | 6 ++-- 4 files changed, 15 insertions(+), 31 deletions(-) create mode 100644 pipeline/test/pipeline/test_pipeline.jl diff --git a/pipeline/src/infer/InferenceConfig.jl b/pipeline/src/infer/InferenceConfig.jl index e7832d9fb..aa28a4964 100644 --- a/pipeline/src/infer/InferenceConfig.jl +++ b/pipeline/src/infer/InferenceConfig.jl @@ -80,6 +80,7 @@ to make inference on and model configuration. """ function create_inference_results(config, epiprob) #Return the sampled infections and observations + idxs = config.tspan[1]:config.tspan[2] y_t = ismissing(config.case_data) ? missing : Vector{Union{Missing, Int64}}(config.case_data[idxs]) inference_results = apply_method(epiprob, @@ -108,7 +109,6 @@ to make inference on and model configuration. function infer(config::InferenceConfig) #Define the EpiProblem epiprob = define_epiprob(config) - idxs = config.tspan[1]:config.tspan[2] #Return the sampled infections and observations inference_results = create_inference_results(config, epiprob) diff --git a/pipeline/test/pipeline/test_pipeline.jl b/pipeline/test/pipeline/test_pipeline.jl new file mode 100644 index 000000000..ad5f71815 --- /dev/null +++ b/pipeline/test/pipeline/test_pipeline.jl @@ -0,0 +1,2 @@ +include("test_pipelinetypes.jl") +include("test_pipelinefunctions.jl") diff --git a/pipeline/test/pipeline/test_pipelinefunctions.jl b/pipeline/test/pipeline/test_pipelinefunctions.jl index 19f9a6bd2..0a49944f3 100644 --- a/pipeline/test/pipeline/test_pipelinefunctions.jl +++ b/pipeline/test/pipeline/test_pipelinefunctions.jl @@ -1,5 +1,5 @@ @testset "do_truthdata tests" begin - using EpiAwarePipeline, Dagger + using Dagger for pipetype in [SmoothOutbreakPipeline, MeasuresOutbreakPipeline, SmoothEndemicPipeline, RoughEndemicPipeline] pipeline = pipetype(; testmode = true) @@ -13,7 +13,7 @@ end @testset "do_inference tests" begin - using EpiAwarePipeline, Dagger, EpiAware + using Dagger function make_inference(pipeline) truthdata_dg_task = do_truthdata(pipeline) @@ -21,23 +21,9 @@ end do_inference(truthdata[1], pipeline) end - pipetype = SmoothOutbreakPipeline - pipeline = pipetype(; ndraws = 40, testmode = true) - truthdata_dg_task = do_truthdata(pipeline) - truthdata = fetch.(truthdata_dg_task) - inference_configs = make_inference_configs(pipeline) - inference_method = make_inference_method(pipeline) - map(inference_configs) do inference_config - generate_inference_results( - truthdata, inference_config, pipeline; inference_method) - end - - inference_results_tsk = make_inference(pipeline) - inference_results = fetch.(inference_results_tsk) - for pipetype in [SmoothOutbreakPipeline, MeasuresOutbreakPipeline, SmoothEndemicPipeline, RoughEndemicPipeline] - pipeline = pipetype(; ndraws = 40, testmode = true) + pipeline = pipetype(; ndraws = 20, nchains = 1, testmode = true) inference_results_tsk = make_inference(pipeline) inference_results = fetch.(inference_results_tsk) @test length(inference_results) == 1 @@ -46,17 +32,13 @@ end end end -@testset "do_pipeline test: just run" begin - using EpiAwarePipeline - pipeline = EpiAwareExamplePipeline() - res = do_pipeline(pipeline) - fetch(res) - @test isnothing(res) -end +@testset "do_pipeline test: just run all pipeline objects" begin + using Dagger + pipelines = map([SmoothOutbreakPipeline, MeasuresOutbreakPipeline, + SmoothEndemicPipeline, RoughEndemicPipeline]) do pipetype + pipetype(; ndraws = 10, nchains = 1, testmode = true) + end -@testset "do_pipeline test: just run as a vector" begin - using EpiAwarePipeline - pipelines = fill(EpiAwareExamplePipeline(), 2) res = do_pipeline(pipelines) fetch(res) @test isnothing(res) diff --git a/pipeline/test/runtests.jl b/pipeline/test/runtests.jl index d565ec9a7..2e111aed5 100644 --- a/pipeline/test/runtests.jl +++ b/pipeline/test/runtests.jl @@ -1,10 +1,9 @@ using DrWatson, Test quickactivate(@__DIR__(), "EpiAwarePipeline") using EpiAwarePipeline, EpiAware - +import Random +Random.seed!(123) # Run tests -include("pipeline/test_pipelinetypes.jl"); -include("pipeline/test_pipelinefunctions.jl"); include("utils/test_utils.jl"); include("constructors/test_constructors.jl"); include("simulate/test_simulate.jl"); @@ -12,3 +11,4 @@ include("infer/test_infer.jl"); include("forecast/test_forecast.jl"); include("scoring/test_score_parameters.jl"); include("plotting/plotting_tests.jl"); +include("pipeline/test_pipeline.jl"); From b47309eb995fb4377f69dc096fc9a9e2b4c2fd98 Mon Sep 17 00:00:00 2001 From: Samuel Brand Date: Fri, 4 Oct 2024 13:56:38 +0100 Subject: [PATCH 2/2] delete stale end-to-end test --- .../test/end-to-end/test_prior_predictive.jl | 125 ------------------ 1 file changed, 125 deletions(-) delete mode 100644 pipeline/test/end-to-end/test_prior_predictive.jl diff --git a/pipeline/test/end-to-end/test_prior_predictive.jl b/pipeline/test/end-to-end/test_prior_predictive.jl deleted file mode 100644 index 8758720e9..000000000 --- a/pipeline/test/end-to-end/test_prior_predictive.jl +++ /dev/null @@ -1,125 +0,0 @@ -using Test, DrWatson -quickactivate(@__DIR__(), "EpiAwarePipeline") - -using EpiAwarePipeline, EpiAware, Plots, Turing -pipetype = [SmoothOutbreakPipeline, MeasuresOutbreakPipeline, - SmoothEndemicPipeline, RoughEndemicPipeline] |> rand - -P = pipetype(; testmode = true, nchains = 1, ndraws = 2000, priorpredictive = true) - -## - -inference_config = make_inference_configs(P) |> first - -missingdata = Dict("y_t" => missing, "I_t" => fill(1.0, 100), "truth_I0" => 1.0, - "truth_gi_mean" => inference_config.gi_mean) -results = generate_inference_results(missingdata, inference_config, P) - -res = results["inference_results"] - -gens = generate_quantiles_for_targets( - res, results["epiprob"].epi_model.data, [0.025, 0.25, 0.5, 0.75, 0.975]) - -gens.log_I_t - -## -using CairoMakie - -function _make_prior_plot_title(config) - igp_str = string(config.igp) - latent_model_str = config.latent_model_name |> uppercase - gi_mean_str = config.gi_mean |> string - T_str = config.tspan[2] |> string - return "Prior pred. IGP: $(igp_str), latent model: $(latent_model_str), truth gi mean: $(gi_mean_str), T: $(T_str)" -end -_make_prior_plot_title(config) - -function _setup_levels(ps) - n_levels = length(ps) - qs = mapreduce(vcat, ps) do percentile - [percentile / 2, 1 - percentile / 2] - end |> x -> [0.5; x] - return qs, n_levels -end -## -# _get_priorpred_plot_title(results["inference_config"]) -## - -function prior_predictive_plot(config, output, epiprob; - ps = [0.05, 0.1, 0.25], - bottom_alpha = 0.1, - top_alpha = 0.5, - case_color = :black, - logI_color = :purple, - rt_color = :blue, - Rt_color = :green, - figsize = (750, 600)) - @assert all(0 .<= ps .< 0.5) "Percentiles must be in the range [0, 0.5)" - prior_pred_plot_title = _make_prior_plot_title(config) - qs, n_levels = _setup_levels(sort(ps)) - opacity_scale = range(bottom_alpha, top_alpha, length = n_levels) |> collect - - # Create the figure and axes - fig = Figure(size = figsize) - ax11 = Axis(fig[1, 1]; xlabel = "t", ylabel = "Cases") - ax12 = Axis(fig[1, 2]; xlabel = "t", ylabel = "log(Incidence)") - ax21 = Axis(fig[2, 1]; xlabel = "t", ylabel = "Exp. growth rate") - ax22 = Axis(fig[2, 2]; xlabel = "t", ylabel = "Reproduction number") - linkxaxes!(ax11, ax21) - linkxaxes!(ax12, ax22) - Label(fig[0, :]; text = prior_pred_plot_title, fontsize = 16) - - # Quantile calculations - gen_y_t = mapreduce(hcat, output.generated) do gen - gen.generated_y_t - end |> X -> timeseries_samples_into_quantiles(X, qs) - gen_quantities = generate_quantiles_for_targets(output, epiprob.epi_model.data, qs) - - # Plot the prior predictive samples - # Cases - f = findfirst(!ismissing, gen_y_t[:, 1]) - lines!(ax11, 1:size(gen_y_t, 1), gen_y_t[:, 1], - color = case_color, linewidth = 3, label = "Median") - for i in 1:n_levels - band!(ax11, f:size(gen_y_t, 1), gen_y_t[f:size(gen_y_t, 1), (2 * i)], - gen_y_t[f:size(gen_y_t, 1), (2 * i) + 1], - color = (case_color, opacity_scale[i]), - label = "($(ps[i]*100)-$((1 - ps[i])*100))%") - end - vlines!(ax11, [f], color = case_color, linestyle = :dash, label = "Obs. window") - axislegend(ax11; position = :lt, framevisible = false) - - # Other quantities - for (ax, target, c) in zip( - [ax12, ax21, ax22], [gen_quantities.log_I_t, gen_quantities.rt, gen_quantities.Rt], - [logI_color, rt_color, Rt_color]) - lines!(ax, 1:size(target, 1), target[:, 1], - color = logI_color, linewidth = 3, label = "Median") - for i in 1:n_levels - band!(ax, 1:size(target, 1), target[:, (2 * i)], target[:, (2 * i) + 1], - color = (c, opacity_scale[i]), label = "") - end - end - - fig -end - -## -fig = prior_predictive_plot(config, res, results["epiprob"]; ps = [0.025, 0.1, 0.25]) -## -gen_y_t = mapreduce(hcat, res.generated) do gen - gen.generated_y_t -end |> X -> timeseries_samples_into_quantiles(X, [0.025, 0.25, 0.5, 0.75, 0.975]) - -## -fig = Figure() -ax_logIt = Axis(fig[1, 1]; - xticks = vcat(1, 5:5:50) # xlabel -) - -for i in 1:5 - lines!(ax_logIt, gen_y_t[:, i], color = :black, alpha = 0.5) -end - -# hlines!(ax, [1.0], color = :red, linestyle = :dash) -fig