diff --git a/PRAS.jl/test/runtests.jl b/PRAS.jl/test/runtests.jl index 7189f62e..db44b014 100644 --- a/PRAS.jl/test/runtests.jl +++ b/PRAS.jl/test/runtests.jl @@ -1,15 +1,49 @@ using PRAS using Test -sys = PRAS.rts_gmlc() +@testset "ShortfallResult" begin + sys = PRAS.rts_gmlc() -sf, = assess(sys, SequentialMonteCarlo(samples=100), Shortfall()) + sf, = assess(sys, SequentialMonteCarlo(samples=100), Shortfall()) -eue = EUE(sf) -lole = LOLE(sf) -neue = NEUE(sf) + eue = EUE(sf) + lole = LOLE(sf) + neue = NEUE(sf) -@test val(eue) isa Float64 -@test stderror(eue) isa Float64 -@test val(neue) isa Float64 -@test stderror(neue) isa Float64 + alpha = 0.95 + cvar = CVAR(sf, alpha) + ncvar = NCVAR(sf, cvar) + + @test val(eue) isa Float64 + @test stderror(eue) isa Float64 + @test val(neue) isa Float64 + @test stderror(neue) isa Float64 + @test val(cvar) isa Float64 + @test stderror(cvar) isa Float64 + @test val(ncvar) isa Float64 + @test stderror(ncvar) isa Float64 + +end + +@testset "ShortfallSamplesResult" begin + sys = PRAS.rts_gmlc() + + sf, = assess(sys, SequentialMonteCarlo(samples=100), ShortfallSamples()) + + eue = EUE(sf) + lole = LOLE(sf) + neue = NEUE(sf) + + alpha = 0.95 + cvar = CVAR(sf, alpha) + ncvar = NCVAR(sf, cvar) + + @test val(eue) isa Float64 + @test stderror(eue) isa Float64 + @test val(neue) isa Float64 + @test stderror(neue) isa Float64 + @test val(cvar) isa Float64 + @test stderror(cvar) isa Float64 + @test val(ncvar) isa Float64 + @test stderror(ncvar) isa Float64 +end \ No newline at end of file diff --git a/PRASCore.jl/src/Results/Results.jl b/PRASCore.jl/src/Results/Results.jl index 3e4e3300..881bc872 100644 --- a/PRASCore.jl/src/Results/Results.jl +++ b/PRASCore.jl/src/Results/Results.jl @@ -4,7 +4,7 @@ import Base: broadcastable, getindex, merge! import OnlineStats: Series import OnlineStatsBase: EqualWeight, Mean, Variance, value import Printf: @sprintf -import StatsBase: mean, std, stderror +import StatsBase: mean, std, stderror, quantile import ..Systems: SystemModel, ZonedDateTime, Period, PowerUnit, EnergyUnit, conversionfactor, @@ -13,7 +13,7 @@ export # Metrics ReliabilityMetric, LOLE, EUE, NEUE, - val, stderror, + val, stderror, CVAR, NCVAR, # Result specifications Shortfall, ShortfallSamples, @@ -80,6 +80,20 @@ NEUE(x::AbstractShortfallResult, r::AbstractString, ::Colon) = NEUE(x::AbstractShortfallResult, ::Colon, ::Colon) = NEUE.(x, x.regions.names, permutedims(x.timestamps)) +CVAR(x::AbstractShortfallResult, alpha::Float64, ::Colon, t::ZonedDateTime) = + CVAR.(x, alpha, x.regions.names, t) + +CVAR(x::AbstractShortfallResult, alpha::Float64, r::AbstractString, ::Colon) = + CVAR.(x, alpha, r, x.timestamps) + +CVAR(x::AbstractShortfallResult, alpha::Float64, ::Colon, ::Colon) = + CVAR.(x, alpha, x.regions.names, permutedims(x.timestamps)) + +NCVAR(x::AbstractShortfallResult, alpha::Float64, r::AbstractString, ::Colon) = + NCVAR.(x, alpha, r, x.timestamps) + +NCVAR(x::AbstractShortfallResult, alpha::Float64, ::Colon, ::Colon) = + NCVAR.(x, alpha, x.regions.names, permutedims(x.timestamps)) include("Shortfall.jl") include("ShortfallSamples.jl") diff --git a/PRASCore.jl/src/Results/Shortfall.jl b/PRASCore.jl/src/Results/Shortfall.jl index 0b00d7d2..1d95fd1a 100644 --- a/PRASCore.jl/src/Results/Shortfall.jl +++ b/PRASCore.jl/src/Results/Shortfall.jl @@ -67,6 +67,10 @@ mutable struct ShortfallAccumulator{S} <: ResultAccumulator{Shortfall} unservedload_total_currentsim::Int unservedload_region_currentsim::Vector{Int} + # Sample-level UE for current simulation + unservedload_sample::Vector{Int} + unservedload_region_sample::Matrix{Int} + end function accumulator( @@ -90,6 +94,8 @@ function accumulator( unservedload_total_currentsim = 0 unservedload_region_currentsim = zeros(Int, nregions) + unservedload_sample = zeros(Int, nsamples) + unservedload_region_sample = zeros(Int, nregions, nsamples) return ShortfallAccumulator{S}( periodsdropped_total, periodsdropped_region, @@ -97,7 +103,8 @@ function accumulator( periodsdropped_total_currentsim, periodsdropped_region_currentsim, unservedload_total, unservedload_region, unservedload_period, unservedload_regionperiod, - unservedload_total_currentsim, unservedload_region_currentsim) + unservedload_total_currentsim, unservedload_region_currentsim, + unservedload_sample, unservedload_region_sample) end @@ -115,6 +122,9 @@ function merge!( foreach(merge!, x.unservedload_period, y.unservedload_period) foreach(merge!, x.unservedload_regionperiod, y.unservedload_regionperiod) + x.unservedload_sample .+= y.unservedload_sample + x.unservedload_region_sample .+= y.unservedload_region_sample + return end @@ -141,14 +151,17 @@ struct ShortfallResult{N, L, T <: Period, E <: EnergyUnit, S} <: eventperiod_regionperiod_mean::Matrix{Float64} eventperiod_regionperiod_std::Matrix{Float64} - shortfall_mean::Matrix{Float64} # r x t + capacity_shortfall_mean::Matrix{Float64} # r x t shortfall_std::Float64 shortfall_region_std::Vector{Float64} shortfall_period_std::Vector{Float64} shortfall_regionperiod_std::Matrix{Float64} + shortfall_samples::Vector{Float64} + shortfall_region_samples::Matrix{Float64} + function ShortfallResult{N,L,T,E,S}( nsamples::Union{Int,Nothing}, regions::Regions, @@ -162,10 +175,13 @@ struct ShortfallResult{N, L, T <: Period, E <: EnergyUnit, S} <: eventperiod_regionperiod_mean::Matrix{Float64}, eventperiod_regionperiod_std::Matrix{Float64}, shortfall_mean::Matrix{Float64}, + capacity_shortfall_mean::Matrix{Float64}, shortfall_std::Float64, shortfall_region_std::Vector{Float64}, shortfall_period_std::Vector{Float64}, - shortfall_regionperiod_std::Matrix{Float64} + shortfall_regionperiod_std::Matrix{Float64}, + shortfall_samples::Vector{Float64}, + shortfall_region_samples::Matrix{Float64}, ) where {N,L,T<:Period,E<:EnergyUnit,S <: Union{Shortfall,DemandResponseShortfall}} isnothing(nsamples) || nsamples > 0 || @@ -185,7 +201,9 @@ struct ShortfallResult{N, L, T <: Period, E <: EnergyUnit, S} <: size(eventperiod_regionperiod_std) == (nregions, N) && length(shortfall_region_std) == nregions && length(shortfall_period_std) == N && - size(shortfall_regionperiod_std) == (nregions, N) || + size(shortfall_regionperiod_std) == (nregions, N) && + size(shortfall_samples) == (nsamples,) && + size(shortfall_region_samples) == (nregions, nsamples) || error("Inconsistent input data sizes") new{N,L,T,E,S}(nsamples, regions, timestamps, @@ -193,9 +211,10 @@ struct ShortfallResult{N, L, T <: Period, E <: EnergyUnit, S} <: eventperiod_region_mean, eventperiod_region_std, eventperiod_period_mean, eventperiod_period_std, eventperiod_regionperiod_mean, eventperiod_regionperiod_std, - shortfall_mean, shortfall_std, + shortfall_mean, capacity_shortfall_mean, shortfall_std, shortfall_region_std, shortfall_period_std, - shortfall_regionperiod_std) + shortfall_regionperiod_std, shortfall_samples, + shortfall_region_samples) end @@ -292,6 +311,90 @@ function NEUE(x::ShortfallResult, r::AbstractString) end +function CVAR(x::ShortfallResult{N,L,T,E}, alpha::Float64) where {N,L,T,E} + + estimate = x.shortfall_samples + var = quantile(estimate, alpha) + tail_losses = estimate[estimate .>= var] + + cvar = if !isempty(tail_losses) + MeanEstimate(tail_losses) + else + MeanEstimate(0.) + end + + capacity_estimate = x.capacity_shortfall_mean[:] + capacity_var = quantile(capacity_estimate, alpha) + capacity_tail_losses = capacity_estimate[capacity_estimate .>= capacity_var] + + capacity_cvar = if !isempty(capacity_tail_losses) + MeanEstimate(capacity_tail_losses) + else + MeanEstimate(0.) + end + + return CVAR{N,L,T,E}(cvar, alpha, var, capacity_cvar, capacity_var) + +end + +function CVAR(x::ShortfallResult{N,L,T,E}, alpha::Float64, r::AbstractString) where {N,L,T,E} + + i_r = findfirstunique(x.regions.names, r) + estimate = x.shortfall_region_samples[i_r, :] + var = quantile(estimate, alpha) + tail_losses = estimate[estimate .>= var] + + cvar = if !isempty(tail_losses) + MeanEstimate(tail_losses) + else + MeanEstimate(0.) + end + + capacity_estimate = x.capacity_shortfall_mean[i_r, :] + capacity_var = quantile(capacity_estimate, alpha) + capacity_tail_losses = capacity_estimate[capacity_estimate .>= capacity_var] + + capacity_cvar = if !isempty(capacity_tail_losses) + MeanEstimate(capacity_tail_losses) + else + MeanEstimate(0.) + end + + return CVAR{N,L,T,E}(cvar, alpha, var, capacity_cvar, capacity_var) + +end + +function NCVAR(x::ShortfallResult{N,L,T,E}, cvar::CVAR) where {N,L,T,E} + demand = sum(x.regions.load) + + if demand > 0 + ncvar = div(cvar.cvar, demand/1e6) + var = div(cvar.var, demand/1e6) + else + ncvar = MeanEstimate(0.) + var = MeanEstimate(0.) + end + + return NCVAR(ncvar, cvar.alpha, var) + +end + +function NCVAR(x::ShortfallResult{N,L,T,E}, cvar::CVAR, r::AbstractString) where {N,L,T,E} + i_r = findfirstunique(x.regions.names, r) + demand = sum(x.regions.load[i_r, :]) + + if demand > 0 + ncvar = div(cvar.cvar, demand/1e6) + var = div(cvar.var, demand/1e6) + else + ncvar = MeanEstimate(0.) + var = MeanEstimate(0.) + end + + return NCVAR(ncvar, cvar.alpha, var) + +end + function finalize( acc::ShortfallAccumulator{S}, system::SystemModel{N,L,T,P,E}, @@ -312,18 +415,22 @@ function finalize( nsamples = first(acc.unservedload_total.stats).n p2e = conversionfactor(L,T,P,E) + capacity_shortfall_mean = ue_regionperiod_mean ue_regionperiod_mean .*= p2e ue_total_std *= p2e ue_region_std .*= p2e ue_period_std .*= p2e ue_regionperiod_std .*= p2e + ue_sample = float(acc.unservedload_sample .* p2e) + ue_region_sample = float(acc.unservedload_region_sample .* p2e) return ShortfallResult{N,L,T,E,S}( nsamples, system.regions, system.timestamps, ep_total_mean, ep_total_std, ep_region_mean, ep_region_std, ep_period_mean, ep_period_std, ep_regionperiod_mean, ep_regionperiod_std, - ue_regionperiod_mean, ue_total_std, - ue_region_std, ue_period_std, ue_regionperiod_std) + ue_regionperiod_mean, capacity_shortfall_mean, ue_total_std, + ue_region_std, ue_period_std, ue_regionperiod_std, + ue_sample, ue_region_sample, ) end diff --git a/PRASCore.jl/src/Results/ShortfallSamples.jl b/PRASCore.jl/src/Results/ShortfallSamples.jl index 99106433..e0ab7342 100644 --- a/PRASCore.jl/src/Results/ShortfallSamples.jl +++ b/PRASCore.jl/src/Results/ShortfallSamples.jl @@ -82,6 +82,7 @@ struct ShortfallSamplesResult{N,L,T<:Period,P<:PowerUnit,E<:EnergyUnit, S} <: Ab timestamps::StepRange{ZonedDateTime,T} shortfall::Array{Int,3} # r x t x s + capacity_shortfall::Array{Int,2} # r x s end @@ -185,12 +186,127 @@ function NEUE(x::ShortfallSamplesResult, r::AbstractString) end +function CVAR(x::ShortfallSamplesResult{N,L,T,P,E}, alpha::Float64) where {N,L,T,P,E} + + estimate = x[] + var = quantile(estimate, alpha) + tail_losses = estimate[estimate .> var] + + cvar = if !isempty(tail_losses) + MeanEstimate(tail_losses) + else + MeanEstimate(0.) + end + + capacity_estimate = x.capacity_shortfall[:] + capacity_var = quantile(capacity_estimate, alpha) + capacity_tail_losses = capacity_estimate[capacity_estimate .>= capacity_var] + + capacity_cvar = if !isempty(capacity_tail_losses) + MeanEstimate(capacity_tail_losses) + else + MeanEstimate(0.) + end + + return CVAR{N,L,T,E}(cvar, alpha, var, capacity_cvar, capacity_var) + +end + +function CVAR(x::ShortfallSamplesResult{N,L,T,P,E}, alpha::Float64, r::AbstractString) where {N,L,T,P,E} + estimate = x[r] + var = quantile(estimate, alpha) + tail_losses = estimate[estimate .>= var] + + cvar = if !isempty(tail_losses) + MeanEstimate(tail_losses) + else + MeanEstimate(0.) + end + + i_r = findfirstunique(x.regions.names, r) + capacity_estimate = x.capacity_shortfall[i_r, :] + capacity_var = quantile(capacity_estimate, alpha) + capacity_tail_losses = capacity_estimate[capacity_estimate .>= capacity_var] + + capacity_cvar = if !isempty(capacity_tail_losses) + MeanEstimate(capacity_tail_losses) + else + MeanEstimate(0.) + end + + return CVAR{N,L,T,E}(cvar, alpha, var, capacity_cvar, capacity_var) + +end + +function CVAR(x::ShortfallSamplesResult{N,L,T,P,E}, alpha::Float64, t::ZonedDateTime) where {N,L,T,P,E} + estimate = x[t] + var = quantile(estimate, alpha) + tail_losses = estimate[estimate .>= var] + + cvar = if !isempty(tail_losses) + MeanEstimate(tail_losses) + else + MeanEstimate(0.) + end + + return CVAR{N,L,T,E}(cvar, alpha, var, cvar, var) + +end + +function CVAR(x::ShortfallSamplesResult{N,L,T,P,E}, alpha::Float64, r::AbstractString, t::ZonedDateTime) where {N,L,T,P,E} + estimate = x[r, t] + var = quantile(estimate, alpha) + tail_losses = estimate[estimate .>= var] + + cvar = if !isempty(tail_losses) + MeanEstimate(tail_losses) + else + MeanEstimate(0.) + end + + return CVAR{N,L,T,E}(cvar, alpha, var, cvar, var) +end + +function NCVAR(x::ShortfallSamplesResult{N,L,T,P}, cvar::CVAR) where {N,L,T,P} + demand = sum(x.regions.load) + + if demand > 0 + ncvar = div(cvar.cvar, demand/1e6) + var = div(cvar.var, demand/1e6) + else + ncvar = MeanEstimate(0.) + var = MeanEstimate(0.) + end + + return NCVAR(ncvar, cvar.alpha, var) + +end + +function NCVAR(x::ShortfallSamplesResult{N,L,T,P}, cvar::CVAR, r::AbstractString) where {N,L,T,P} + i_r = findfirstunique(x.regions.names, r) + demand = sum(x.regions.load[i_r, :]) + + if demand > 0 + ncvar = div(cvar.cvar, demand/1e6) + var = div(cvar.var, demand/1e6) + else + ncvar = MeanEstimate(0.) + var = MeanEstimate(0.) + end + + return NCVAR(ncvar, cvar.alpha, var) + +end + function finalize( acc::ShortfallSamplesAccumulator{S}, system::SystemModel{N,L,T,P,E}, ) where {N,L,T,P,E,S<:Union{ShortfallSamples,DemandResponseShortfallSamples}} + n_regions = length(system.regions) + p2e = conversionfactor(L,T,P,E) + max_capacity_shortfall = reshape(maximum(acc.shortfall, dims=[2]) ./ p2e, n_regions, :) return ShortfallSamplesResult{N,L,T,P,E,S}( - system.regions, system.timestamps, acc.shortfall) + system.regions, system.timestamps, acc.shortfall, max_capacity_shortfall) end diff --git a/PRASCore.jl/src/Results/metrics.jl b/PRASCore.jl/src/Results/metrics.jl index f44c7096..91a5dbc1 100644 --- a/PRASCore.jl/src/Results/metrics.jl +++ b/PRASCore.jl/src/Results/metrics.jl @@ -22,7 +22,11 @@ MeanEstimate(mu::Real, sigma::Real, n::Int) = MeanEstimate(mu, sigma / sqrt(n)) function MeanEstimate(xs::AbstractArray{<:Real}) est = mean(xs) - return MeanEstimate(est, std(xs, mean=est), length(xs)) + if length(xs) > 1 + MeanEstimate(est, std(xs, mean=est), length(xs)) + else + MeanEstimate(est) + end end val(est::MeanEstimate) = est.estimate @@ -164,3 +168,73 @@ function Base.show(io::IO, x::NEUE) print(io, "NEUE = ", x.neue, " ppm") end + +""" + CVAR + +`CVAR` reports conditional value at risk of shortfalls, for total unserved energy and capacity shortfalls. + +Contains both the estimated value itself as well as the standard error +of that estimate, which can be extracted with `val` and `stderror`, +respectively. +""" +struct CVAR{N,L,T<:Period,E<:EnergyUnit} <: ReliabilityMetric + + cvar::MeanEstimate + alpha::Float64 + var::Float64 + capacity_cvar::MeanEstimate + capacity_var::Float64 + + function CVAR{N,L,T,E}(cvar::MeanEstimate, + alpha::Float64, + var::Float64, + capacity_cvar::MeanEstimate, + capacity_var::Float64) where {N,L,T<:Period,E<:EnergyUnit} + val(cvar) >= 0 || throw(DomainError( + "$val is not a valid CVAR")) + new{N,L,T,E}(cvar, alpha, var, capacity_cvar, capacity_var) + end + +end + +val(x::CVAR) = val(x.cvar) +stderror(x::CVAR) = stderror(x.cvar) + +function Base.show(io::IO, x::CVAR{N,L,T,E}) where {N,L,T,E} + + print(io, "CVAR@$(x.alpha) = ", x.cvar, " ", + unitsymbol(E), "/", N*L == 1 ? "" : N*L, unitsymbol(T)) + +end + +""" + NCVAR + +`NCVAR` reports normalized conditional value at risk of shortfalls, for total unserved energy and capacity shortfalls. + +Contains both the estimated value itself as well as the standard error +of that estimate, which can be extracted with `val` and `stderror`, +respectively. +""" +struct NCVAR <: ReliabilityMetric + + ncvar::MeanEstimate + alpha::Float64 + var::Float64 + + function NCVAR(ncvar::MeanEstimate, alpha::Float64, var::Float64) + val(ncvar) >= 0 || throw(DomainError( + "$val is not a valid NCVAR")) + new(ncvar, alpha, var) + end + +end + +val(x::NCVAR) = val(x.ncvar) +stderror(x::NCVAR) = stderror(x.ncvar) + +function Base.show(io::IO, x::NCVAR) + print(io, "NCVAR@$(x.alpha) = ", x.ncvar, " ppm") + +end \ No newline at end of file diff --git a/PRASCore.jl/src/Simulations/recording.jl b/PRASCore.jl/src/Simulations/recording.jl index a156eb32..44f6ff7d 100644 --- a/PRASCore.jl/src/Simulations/recording.jl +++ b/PRASCore.jl/src/Simulations/recording.jl @@ -55,7 +55,9 @@ function record!( end function reset!(acc::Results.ShortfallAccumulator, sampleid::Int) - + # Store total UE for each sample + acc.unservedload_sample[sampleid] = acc.unservedload_total_currentsim + # Store regional / total sums for current simulation fit!(acc.periodsdropped_total, acc.periodsdropped_total_currentsim) fit!(acc.unservedload_total, acc.unservedload_total_currentsim) @@ -63,6 +65,7 @@ function reset!(acc::Results.ShortfallAccumulator, sampleid::Int) for r in eachindex(acc.periodsdropped_region) fit!(acc.periodsdropped_region[r], acc.periodsdropped_region_currentsim[r]) fit!(acc.unservedload_region[r], acc.unservedload_region_currentsim[r]) + acc.unservedload_region_sample[r, sampleid] = acc.unservedload_region_currentsim[r] end # Reset for new simulation diff --git a/PRASCore.jl/src/Systems/TestData.jl b/PRASCore.jl/src/Systems/TestData.jl index 6b5ba264..07c2cfc2 100644 --- a/PRASCore.jl/src/Systems/TestData.jl +++ b/PRASCore.jl/src/Systems/TestData.jl @@ -36,6 +36,7 @@ singlenode_a_lole = 0.355 singlenode_a_lolps = [0.028, 0.271, 0.028, 0.028] singlenode_a_eue = 1.59 singlenode_a_eues = [0.29, 0.832, 0.29, 0.178] +singlenode_a_cvar = 11.84 ## Single-Region System A - 5 minute version @@ -99,6 +100,7 @@ singlenode_b_lole = 0.96 singlenode_b_lolps = [0.19, 0.19, 0.19, 0.1, 0.1, 0.19] singlenode_b_eue = 7.11 singlenode_b_eues = [1.29, 1.29, 1.29, 0.85, 1.05, 1.34] +singlenode_b_cvar = 30.44 # Single-Region System B, with storage diff --git a/PRASCore.jl/test/Results/metrics.jl b/PRASCore.jl/test/Results/metrics.jl index fb7bf796..08232c04 100644 --- a/PRASCore.jl/test/Results/metrics.jl +++ b/PRASCore.jl/test/Results/metrics.jl @@ -72,4 +72,28 @@ end + @testset "CVAR" begin + + cvar1 = CVAR{2,1,Hour,MWh}(MeanEstimate(1.2), 0.95, 1.2, MeanEstimate(1.0), 1.2) + @test string(cvar1) == "CVAR@0.95 = 1.20000 MWh/2h" + + cvar2 = CVAR{1,2,Year,GWh}(MeanEstimate(17.2, 1.3), 0.95, 1.2, MeanEstimate(17.2, 1.3), 1.2) + @test string(cvar2) == "CVAR@0.95 = 17±1 GWh/2y" + + @test_throws DomainError CVAR{1,1,Hour,MWh}(MeanEstimate(-1.2), 0.95, 1.2, MeanEstimate(-1.2), 1.2) + + end + + @testset "NCVAR" begin + + ncvar1 = NCVAR(MeanEstimate(1.2), 0.95, 1.2) + @test string(ncvar1) == "NCVAR@0.95 = 1.20000 ppm" + + ncvar2 = NCVAR(MeanEstimate(17.2, 1.3), 0.95, 1.3) + @test string(ncvar2) == "NCVAR@0.95 = 17±1 ppm" + + @test_throws DomainError NCVAR(MeanEstimate(-1.2), 0.95, -1.2) + + end + end diff --git a/PRASCore.jl/test/Results/shortfall.jl b/PRASCore.jl/test/Results/shortfall.jl index 9f5d4a01..c9df817e 100644 --- a/PRASCore.jl/test/Results/shortfall.jl +++ b/PRASCore.jl/test/Results/shortfall.jl @@ -4,13 +4,15 @@ N = DD.nperiods r, r_idx, r_bad = DD.testresource, DD.testresource_idx, DD.notaresource t, t_idx, t_bad = DD.testperiod, DD.testperiod_idx, DD.notaperiod + alpha = DD.alpha result = PRASCore.Results.ShortfallResult{N,1,Hour,MWh,Shortfall}( DD.nsamples, Regions{N,MW}(DD.resourcenames, DD.resource_vals), DD.periods, DD.d1, DD.d2, DD.d1_resource, DD.d2_resource, DD.d1_period, DD.d2_period, DD.d1_resourceperiod, DD.d2_resourceperiod, - DD.d3_resourceperiod, - DD.d4, DD.d4_resource, DD.d4_period, DD.d4_resourceperiod) + DD.d3_resourceperiod, DD.d3_resourceperiod, + DD.d4, DD.d4_resource, DD.d4_period, DD.d4_resourceperiod, + DD.d1_sample, DD.d1_resourcesample) # Overall @@ -28,6 +30,25 @@ load = sum(DD.resource_vals) @test val(neue) ≈ first(result[]) / load*1e6 @test stderror(neue) ≈ last(result[]) / sqrt(DD.nsamples) / load*1e6 + + cvar = CVAR(result, alpha) + estimate = result.shortfall_samples; + tail_losses = estimate[estimate .>= quantile(estimate, alpha)]; + @test val(cvar) ≈ mean(tail_losses) + @test stderror(cvar) ≈ std(tail_losses) / sqrt(length(tail_losses)) + + capacity_cvar = cvar.capacity_cvar + cap_shortfal = vec(reshape(result.capacity_shortfall_mean, 1, :)) + var = quantile(cap_shortfal, alpha) + tail_losses = cap_shortfal[cap_shortfal .>= var] + capacity_cvar_check = mean(tail_losses) + @test val(capacity_cvar) ≈ capacity_cvar_check + @test stderror(capacity_cvar) ≈ std(tail_losses) / sqrt(length(tail_losses)) + + ncvar = NCVAR(result, cvar) + @test val(ncvar) ≈ val(cvar) / load*1e6 + @test stderror(ncvar) ≈ stderror(cvar) / load*1e6 + # Region-specific @test result[r] ≈ (sum(DD.d3_resourceperiod[r_idx,:]), DD.d4_resource[r_idx]) @@ -45,10 +66,30 @@ @test val(region_neue) ≈ first(result[r]) / load*1e6 @test stderror(region_neue) ≈ last(result[r]) / sqrt(DD.nsamples) / load*1e6 + region_cvar = CVAR(result, alpha, r) + region_estimate = result.shortfall_region_samples[r_idx, :]; + region_tail_losses = region_estimate[region_estimate .>= quantile(region_estimate, alpha)]; + @test val(region_cvar) ≈ mean(region_tail_losses) + @test stderror(region_cvar) ≈ std(region_tail_losses) / sqrt(length(region_tail_losses)) + + region_capacity_cvar = region_cvar.capacity_cvar + region_cap_shortfal = result.capacity_shortfall_mean[r_idx, :] + var = quantile(region_cap_shortfal, alpha) + tail_losses = region_cap_shortfal[region_cap_shortfal .>= var] + region_capacity_cvar_check = mean(tail_losses) + @test val(region_capacity_cvar) ≈ region_capacity_cvar_check + @test stderror(region_capacity_cvar) ≈ std(tail_losses) / sqrt(length(tail_losses)) + + region_ncvar = NCVAR(result, region_cvar, r) + @test val(region_ncvar) ≈ val(region_cvar) / load*1e6 + @test stderror(region_ncvar) ≈ stderror(region_cvar) / load*1e6 + @test_throws BoundsError result[r_bad] @test_throws BoundsError LOLE(result, r_bad) @test_throws BoundsError EUE(result, r_bad) @test_throws BoundsError NEUE(result, r_bad) + @test_throws BoundsError CVAR(result, alpha, r_bad) + @test_throws BoundsError NCVAR(result, region_cvar, r_bad) # Period-specific @@ -100,9 +141,10 @@ end N = DD.nperiods r, r_idx, r_bad = DD.testresource, DD.testresource_idx, DD.notaresource t, t_idx, t_bad = DD.testperiod, DD.testperiod_idx, DD.notaperiod + alpha = 0.95 result = PRASCore.Results.ShortfallSamplesResult{N,1,Hour,MW,MWh,ShortfallSamples}( - Regions{N,MW}(DD.resourcenames, DD.resource_vals), DD.periods, DD.d) + Regions{N,MW}(DD.resourcenames, DD.resource_vals), DD.periods, DD.d, DD.cap_d) # Overall @@ -123,6 +165,24 @@ end @test val(neue) ≈ mean(result[]) / load*1e6 @test stderror(neue) ≈ std(result[]) / sqrt(DD.nsamples) / load*1e6 + cvar = CVAR(result, alpha) + estimate = result[]; + tail_losses = estimate[estimate .>= quantile(estimate, alpha)]; + @test val(cvar) ≈ mean(tail_losses) + @test stderror(cvar) ≈ std(tail_losses) / sqrt(length(tail_losses)) + + capacity_cvar = cvar.capacity_cvar + cap_shortfal = vec(reshape(result.capacity_shortfall, 1, :)) + var = quantile(cap_shortfal, alpha) + tail_losses = cap_shortfal[cap_shortfal .>= var] + capacity_cvar_check = mean(tail_losses) + @test val(capacity_cvar) ≈ capacity_cvar_check + @test stderror(capacity_cvar) ≈ std(tail_losses) / sqrt(length(tail_losses)) + + ncvar = NCVAR(result, cvar) + @test val(ncvar) ≈ val(cvar) / load*1e6 + @test stderror(ncvar) ≈ stderror(cvar) / load*1e6 + # Region-specific @test length(result[r]) == DD.nsamples @@ -142,10 +202,30 @@ end @test val(region_neue) ≈ mean(result[r]) / load*1e6 @test stderror(region_neue) ≈ std(result[r]) / sqrt(DD.nsamples) / load*1e6 + region_cvar = CVAR(result, alpha, r) + region_estimate = result[r]; + region_tail_losses = region_estimate[region_estimate .>= quantile(region_estimate, alpha)]; + @test val(region_cvar) ≈ mean(region_tail_losses) + @test stderror(region_cvar) ≈ std(region_tail_losses) / sqrt(length(region_tail_losses)) + + region_capacity_cvar = region_cvar.capacity_cvar + region_cap_shortfal = result.capacity_shortfall[r_idx, :] + var = quantile(region_cap_shortfal, alpha) + tail_losses = region_cap_shortfal[region_cap_shortfal .>= var] + region_capacity_cvar_check = mean(tail_losses) + @test val(region_capacity_cvar) ≈ region_capacity_cvar_check + @test stderror(region_capacity_cvar) ≈ std(tail_losses) / sqrt(length(tail_losses)) + + region_ncvar = NCVAR(result, region_cvar, r) + @test val(region_ncvar) ≈ val(region_cvar) / load*1e6 + @test stderror(region_ncvar) ≈ stderror(region_cvar) / load*1e6 + @test_throws BoundsError result[r_bad] @test_throws BoundsError LOLE(result, r_bad) @test_throws BoundsError EUE(result, r_bad) @test_throws BoundsError NEUE(result, r_bad) + @test_throws BoundsError CVAR(result, alpha, r_bad) + @test_throws BoundsError NCVAR(result, region_cvar, r_bad) # Period-specific @@ -161,9 +241,16 @@ end @test val(period_eue) ≈ mean(result[t]) @test stderror(period_eue) ≈ std(result[t]) / sqrt(DD.nsamples) + period_cvar = CVAR(result, alpha, t) + period_estimate = result[t]; + period_tail_losses = period_estimate[period_estimate .>= quantile(period_estimate, alpha)]; + @test val(period_cvar) ≈ mean(period_tail_losses) + @test stderror(period_cvar) ≈ std(period_tail_losses) / sqrt(length(period_tail_losses)) + @test_throws BoundsError result[t_bad] @test_throws BoundsError LOLE(result, t_bad) @test_throws BoundsError EUE(result, t_bad) + @test_throws BoundsError CVAR(result, alpha, t_bad) # Region + period-specific @@ -180,6 +267,12 @@ end @test val(regionperiod_eue) ≈ mean(result[r, t]) @test stderror(regionperiod_eue) ≈ std(result[r, t]) / sqrt(DD.nsamples) + regionperiod_cvar = CVAR(result, alpha, r, t) + regionperiod_estimate = result[r, t]; + regionperiod_tail_losses = regionperiod_estimate[regionperiod_estimate .>= quantile(regionperiod_estimate, alpha)]; + @test val(regionperiod_cvar) ≈ mean(regionperiod_tail_losses) + @test stderror(regionperiod_cvar) ≈ std(regionperiod_tail_losses) / sqrt(length(regionperiod_tail_losses)) + @test_throws BoundsError result[r, t_bad] @test_throws BoundsError result[r_bad, t] @test_throws BoundsError result[r_bad, t_bad] @@ -192,4 +285,8 @@ end @test_throws BoundsError EUE(result, r_bad, t) @test_throws BoundsError EUE(result, r_bad, t_bad) + @test_throws BoundsError CVAR(result, alpha, r, t_bad) + @test_throws BoundsError CVAR(result, alpha, r_bad, t) + @test_throws BoundsError CVAR(result, alpha, r_bad, t_bad) + end diff --git a/PRASCore.jl/test/Simulations/runtests.jl b/PRASCore.jl/test/Simulations/runtests.jl index 3c518634..fe1ebf8f 100644 --- a/PRASCore.jl/test/Simulations/runtests.jl +++ b/PRASCore.jl/test/Simulations/runtests.jl @@ -5,6 +5,7 @@ end nstderr_tol = 3 + alpha = 0.95 simspec = SequentialMonteCarlo(samples=100_000, seed=1, threaded=false) smallsample = SequentialMonteCarlo(samples=10, seed=123, threaded=false) @@ -66,6 +67,8 @@ TestData.singlenode_a_lole, nstderr_tol) @test withinrange(EUE(shortfall_1a), TestData.singlenode_a_eue, nstderr_tol) + @test withinrange(CVAR(shortfall_1a, alpha), + TestData.singlenode_a_cvar, nstderr_tol) @test withinrange(LOLE(shortfall_1a, "Region"), TestData.singlenode_a_lole, nstderr_tol) @test withinrange(EUE(shortfall_1a, "Region"), @@ -134,10 +137,14 @@ TestData.singlenode_b_lole, nstderr_tol) @test withinrange(EUE(shortfall_1b), TestData.singlenode_b_eue, nstderr_tol) + @test withinrange(CVAR(shortfall_1b, alpha), + TestData.singlenode_b_cvar, nstderr_tol) @test withinrange(LOLE(shortfall_1b, "Region"), TestData.singlenode_b_lole, nstderr_tol) @test withinrange(EUE(shortfall_1b, "Region"), TestData.singlenode_b_eue, nstderr_tol) + @test withinrange(CVAR(shortfall_1b, alpha, "Region"), + TestData.singlenode_b_cvar, nstderr_tol) @test all(LOLE.(shortfall_1b, timestamps_b) .≈ LOLE.(shortfall2_1b, timestamps_b)) diff --git a/PRASCore.jl/test/dummydata.jl b/PRASCore.jl/test/dummydata.jl index 06f7d139..12f96184 100644 --- a/PRASCore.jl/test/dummydata.jl +++ b/PRASCore.jl/test/dummydata.jl @@ -6,6 +6,7 @@ using TimeZones const tz = tz"UTC" nsamples = 100 +alpha = 0.95 resourcenames = ["A", "B", "C"] nresources = length(resourcenames) @@ -27,11 +28,14 @@ testperiod = periods[testperiod_idx] notaperiod = ZonedDateTime(2010,1,1,0,tz) d = rand(0:999, nresources, nperiods, nsamples) +cap_d = rand(0:999, nresources, nsamples) d1 = rand() d1_resource = rand(nresources) d1_period = rand(nperiods) d1_resourceperiod = rand(nresources, nperiods) +d1_sample = rand(nsamples) * 999 +d1_resourcesample = rand(nresources, nsamples) * 999 d2 = rand() d2_resource = rand(nresources)