From df6dc706c11cd324b78b9d6607ea98228f708b6c Mon Sep 17 00:00:00 2001
From: orichters <oliverr@pik-potsdam.de>
Date: Fri, 27 Sep 2024 17:49:36 +0200
Subject: [PATCH] improve checkProjectSummations for ELEVATE + ScenarioMIP

---
 DESCRIPTION                                   |  2 +-
 .../output/single/checkProjectSummations.R    | 68 ++++++++++---------
 2 files changed, 36 insertions(+), 34 deletions(-)

diff --git a/DESCRIPTION b/DESCRIPTION
index d10f40021..66212dc6d 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -54,7 +54,7 @@ Imports:
     nleqslv,
     optparse,
     piamenv (>= 0.4.0),
-    piamInterfaces (>= 0.31.0),
+    piamInterfaces (>= 0.34.1),
     piamPlotComparison (>= 0.0.10),
     piamutils,
     plotly,
diff --git a/scripts/output/single/checkProjectSummations.R b/scripts/output/single/checkProjectSummations.R
index d2ff53bd1..bf4ca5d50 100644
--- a/scripts/output/single/checkProjectSummations.R
+++ b/scripts/output/single/checkProjectSummations.R
@@ -28,56 +28,58 @@ options(width = 160)
 absDiff <- 0.00001
 relDiff <- 0.01
 
-# emi variables where bunkers are added only to the World level
-gases <- c("BC", "CO", "CO2", "Kyoto Gases", "NOx", "OC", "Sulfur", "VOC")
-vars <- c("", "|Energy", "|Energy Demand|Transportation", "|Energy and Industrial Processes",
-          "|Energy|Demand", "|Energy|Demand|Transportation")
-gasvars <- expand.grid(gases, vars, stringsAsFactors = FALSE)
-bunkervars <- unique(sort(c("Gross Emissions|CO2", paste0("Emissions|", gasvars$Var1, gasvars$Var2))))
+sources <- paste0("RT", if (any(grepl("^MAgPIE", levels(mifdata$model)))) "M")
+message("\n### Check existence of variables in mappings.")
+missingVariables <- checkMissingVars(mifdata, TRUE, sources)
+if (length(missingVariables) > 0) message("Check piamInterfaces::variableInfo('variablename') etc.")
 
-for (mapping in c("AR6", "NAVIGATE")) {
-  message("\n### Check project summations for ", mapping)
-  mappingVariables <- mapping %>%
-    getMappingVariables(paste0("RT", if (any(grepl("^MAgPIE", levels(mifdata$model)))) "M")) %>%
-    unique() %>%
-    deletePlus()
-  computedVariables <- unique(paste0(deletePlus(mifdata$variable), " (", gsub("^$", "unitless", mifdata$unit), ")"))
-  missingVariables <- sort(setdiff(mappingVariables, computedVariables))
-  if (length(missingVariables) > 0) {
-    message("# The following ", length(missingVariables), " variables are expected in the piamInterfaces package ",
-            "for mapping ", mapping, ", but cannot be found in the reporting:\n- ",
-            paste(missingVariables, collapse = ",\n- "), "\n")
+checkMappings <- list( # list(mappings, summationsFile, skipBunkers)
+  list(c("AR6", "AR6_NGFS"), "AR6", TRUE),
+  list(c("NAVIGATE", "ELEVATE"), "NAVIGATE", FALSE),
+  list("ScenarioMIP", NULL, FALSE)
+)
+
+for (i in seq_along(checkMappings)) {
+  mapping <- checkMappings[[i]][[1]]
+  message("\n### Check project summations for ", paste(mapping, collapse = ", "))
+  # checkMissingVars
+  checkMissingVars(mifdata, mapping, sources)
+
+  # generate IIASASubmission
+  d <- generateIIASASubmission(mifdata, outputDirectory = NULL, outputFilename = NULL, logFile = NULL,
+                               mapping = mapping, checkSummation = FALSE, generatePlots = FALSE)
+  # Check variable summation, but using only the first mapping
+  failvars <- data.frame()
+  if (length(checkMappings[[i]][[2]]) > 0) {
+    failvars <- d %>%
+      checkSummations(template = mapping[[1]], summationsFile = checkMappings[[i]][[2]], logFile = NULL,
+                      dataDumpFile = NULL, absDiff = absDiff, relDiff = relDiff) %>%
+      filter(abs(diff) >= absDiff, abs(reldiff) >= relDiff) %>%
+      df_variation() %>%
+      droplevels()
   }
 
-  d <- generateIIASASubmission(mifdata, outputDirectory = NULL, logFile = NULL,
-                               mapping = mapping, checkSummation = FALSE)
-  failvars <- d %>%
-    checkSummations(template = mapping, summationsFile = mapping, logFile = NULL, dataDumpFile = NULL,
-                    absDiff = absDiff, relDiff = relDiff) %>%
-    filter(abs(diff) >= absDiff, abs(reldiff) >= relDiff) %>%
-    df_variation() %>%
-    droplevels()
-  
   csregi <- d %>%
     filter(.data$region %in% unique(c("GLO", "World", read.csv2(envi$cfg$regionmapping)$RegionCode))) %>%
-    checkSummationsRegional(intensiveUnits = TRUE) %>%
+    checkSummationsRegional(intensiveUnits = TRUE, skipBunkers = isTRUE(checkMappings[[i]][[3]])) %>%
     rename(World = "total") %>%
     droplevels()
   checkyear <- 2050
   failregi <- csregi %>%
     filter(abs(.data$reldiff) > 0.5, abs(.data$diff) > 0.00015, period == checkyear) %>%
-    filter(! .data$variable %in% bunkervars) %>%
     select(-"model", -"scenario")
   if (nrow(failregi) > 0) {
-    message("For those ", mapping, " variables, the sum of regional values does not match the World value in 2050:")
+    message("For those variables from ", paste(mapping, collapse = ", "),
+            ", the sum of regional values does not match the World value in 2050:")
     failregi %>% piamInterfaces::niceround() %>% print(n = 1000)
   } else {
     message("Regional summation checks are fine.")
   }
 
-  if (nrow(failvars) > 0 || nrow(failregi) > 0 || length(missingVariables) > 0) stopmessage <- c(stopmessage, mapping)
+  if (nrow(failvars) > 0 || nrow(failregi) > 0 || length(missingVariables) > 0) stopmessage <- c(stopmessage, paste(mapping, collapse = "+"))
 }
 
-if (length(stopmessage) > 0) {
-  stop("Failing summation checks for ", paste(stopmessage, collapse = ", "), ", see above.")
+if (length(stopmessage) > 0 || length(missingVariables) > 0) {
+  stop("Project-related issues found checks for ", paste(stopmessage, collapse = ", "), " and ",
+       length(missingVariables), " missing variables found, see above.")
 }