[core, tests, docs] add remaining multi-layer support

avaframe/ana4Stats/getStats.py

@@ -21,7 +21,16 @@
 log = logging.getLogger(__name__)
 
 
-def extractMaxValues(inputDir, avaDir, varPar, restrictType='', nameScenario='', parametersDict=''):
+def extractMaxValues(
+    inputDir,
+    avaDir,
+    varPar,
+    restrictType="",
+    nameScenario="",
+    parametersDict="",
+    layer="",
+    modName="com1DFA",
+):
     """ Extract max values of result parameters and save to dictionary
 
         - optionally restrict data of peak fields by defining which result parameter with restrictType,
@@ -51,12 +60,25 @@ def extractMaxValues(inputDir, avaDir, varPar, restrictType='', nameScenario='',
         """
 
     # filter simulation results using parametersDict
-    simNameList = cfgHandling.filterSims(avaDir, parametersDict)
+    simNameList = cfgHandling.filterSims(avaDir, parametersDict, modName=modName)
     # load dataFrame of all simulation configurations
-    simDF = cfgUtils.createConfigurationInfo(avaDir, standardCfg='')
+    simDF = cfgUtils.createConfigurationInfo(avaDir, standardCfg="", comModule=modName)
 
     # load peakFiles of all simulations and generate dataframe
     peakFilesDF = fU.makeSimDF(inputDir, avaDir=avaDir)
+    hasMultiLayer = any(peakFilesDF["layer"] != "")
+    if hasMultiLayer and layer == "":
+        message = (
+            "Multi-layer results detected but no layer specified. "
+            "Set 'layer' in getStatsCfg.ini (e.g. layer = L1)"
+        )
+        log.error(message)
+        raise ValueError(message)
+    elif not hasMultiLayer and layer != "":
+        message = "No multi-layer results detected but layer is specified. "
+        log.error(message)
+        raise ValueError(message)
+
     nSims = len(peakFilesDF['simName'])
     # initialize peakValues dictionary
     peakValues = {}
@@ -65,60 +87,79 @@ def extractMaxValues(inputDir, avaDir, varPar, restrictType='', nameScenario='',
 
     # Loop through result field files and compute statistical measures
     for m in range(nSims):
-        # filter simulations according to parametersDict
-        if peakFilesDF['simName'][m] in simNameList:
-
-            # Load data
-            fileName = peakFilesDF['files'][m]
-            simName = peakFilesDF['simName'][m]
-            dataFull = IOf.readRaster(fileName, noDataToNan=True)
-
-            # if restrictType, set result field values to nan if restrictType result field equals 0
-            if restrictType != '':
-                peakFilesSimName = peakFilesDF[peakFilesDF['simName'] == simName]
-                fileNameRestrict = peakFilesSimName[peakFilesSimName['resType'] == restrictType]['files'].values[0]
-                dataRestrict = IOf.readRaster(fileNameRestrict, noDataToNan=True)
-                data = np.where((dataRestrict['rasterData'] == 0.0), np.nan, dataFull['rasterData'])
-            else:
-                data = dataFull['rasterData']
-
-            # compute max, mean, min and standard deviation of result field
-            max = np.nanmax(data)
-            min = np.nanmin(data)
-            mean = np.nanmean(data)
-            std = np.nanstd(data)
-            statVals = {'max': max, 'min': min, 'mean': mean, 'std': std}
-            # add statistical measures
-            peakValues[simName].update({peakFilesDF['resType'][m]: statVals})
-
-            # fetch varPar value and nameScenario
-            varParVal = simDF[simDF['simName'] == simName][varPar].iloc[0]
-
-            # if varParVal is empty or nan check if thickness value
-            # if so - read from shp thickness value for each feature need to be found
-            if (varParVal == '' or pd.isna(varParVal)) and varPar in ['relTh', 'entTh', 'secondaryRelTh']:
-                # fetch id of thickness features
-                varParId = varPar + 'Id'
-                varParIds = str(simDF[simDF['simName'] == simName][varParId].iloc[0])
-                varParIdList = varParIds.split('|')
-                # create feature thickness parameter names
-                varParFirstName = varPar + varParIdList[0]
-                # chose value from first feature found!
-                varParVal = simDF[simDF['simName'] == simName][varParFirstName].iloc[0]
-                if len(varParIdList) > 1:
-                    log.warning('%s value - there are multiple features - %s value used '
-                                'from first feature only!' % (varPar, varPar))
-            # cadd varPar value to dictionary
-            peakValues[simName].update({'varPar': varParVal})
-            if nameScenario != '':
-                nameScenarioVal = simDF[simDF['simName'] == simName][nameScenario]
-                peakValues[simName].update({'scenario': nameScenarioVal.iloc[0]})
-                log.info('Simulation parameter %s= %s for resType: %s and name %s' %
-                        (varPar, str(varParVal), peakFilesDF['resType'][m], nameScenarioVal.iloc[0]))
-            else:
-                log.info('Simulation parameter %s= %s for resType: %s' %
-                        (varPar, str(varParVal), peakFilesDF['resType'][m]))
+        simName = peakFilesDF['simName'][m]
+        # skip simulations not in the filtered list
+        if simName not in simNameList:
+            peakValues.pop(simName, None)
+            continue
+
+        # Load data
+        fileName = peakFilesDF['files'][m]
+        dataFull = IOf.readRaster(fileName, noDataToNan=True)
+
+        # if restrictType, set result field values to nan if restrictType result field equals 0
+        if restrictType != "":
+            peakFilesSimName = peakFilesDF[peakFilesDF['simName'] == simName]
+            # mask for resultType
+            mask = peakFilesSimName["resType"] == restrictType
+            # mask for layer
+            if layer:
+                mask &= peakFilesSimName["layer"] == layer
+            fileNameRestrict1 = peakFilesSimName[mask]
+            if len(fileNameRestrict1) == 0:
+                message = "No %s file found for %s to restrict current result file %s with." % (
+                    restrictType,
+                    simName,
+                    fileName.name,
+                )
+                log.error(message)
+                raise ValueError(message)
+            fileNameRestrict = fileNameRestrict1["files"].to_list()[0]
+            dataRestrict = IOf.readRaster(fileNameRestrict, noDataToNan=True)
+            log.info("%s file restricted using %s." % (fileName.name, fileNameRestrict.name))
+            data = np.where((dataRestrict['rasterData'] == 0.0), np.nan, dataFull['rasterData'])
+        else:
+            data = dataFull['rasterData']
+
+        # compute max, mean, min and standard deviation of result field
+        max = np.nanmax(data)
+        min = np.nanmin(data)
+        mean = np.nanmean(data)
+        std = np.nanstd(data)
+        statVals = {'max': max, 'min': min, 'mean': mean, 'std': std}
+        # use layer-suffixed key for multi-layer results (e.g. ppr_l1)
+        resType = peakFilesDF['resType'][m]
+        if peakFilesDF['layer'][m]:
+            resType = "%s_%s" % (resType, peakFilesDF['layer'][m].lower())
+        peakValues[simName].update({resType: statVals})
+
+        # fetch varPar value and nameScenario
+        simRow = simDF[simDF['simName'] == simName]
+        varParVal = simRow[varPar].iloc[0]
+
+        # if varParVal is empty or nan check if thickness value
+        # if so - read from shp thickness value for each feature need to be found
+        if (varParVal == '' or pd.isna(varParVal)) and varPar in ['relTh', 'entTh', 'secondaryRelTh']:
+            # fetch id of thickness features
+            varParId = varPar + 'Id'
+            varParIds = str(simRow[varParId].iloc[0])
+            varParIdList = varParIds.split('|')
+            # create feature thickness parameter names
+            varParFirstName = varPar + varParIdList[0]
+            # chose value from first feature found!
+            varParVal = simRow[varParFirstName].iloc[0]
+            if len(varParIdList) > 1:
+                log.warning('%s value - there are multiple features - %s value used '
+                            'from first feature only!' % (varPar, varPar))
+        # add varPar value to dictionary
+        peakValues[simName].update({'varPar': varParVal})
+        if nameScenario != '':
+            nameScenarioVal = simRow[nameScenario]
+            peakValues[simName].update({'scenario': nameScenarioVal.iloc[0]})
+            log.info('Simulation parameter %s= %s for resType: %s and name %s' %
+                    (varPar, str(varParVal), peakFilesDF['resType'][m], nameScenarioVal.iloc[0]))
         else:
-            peakValues.pop(peakFilesDF['simName'][m], None)
+            log.info('Simulation parameter %s= %s for resType: %s' %
+                    (varPar, str(varParVal), peakFilesDF['resType'][m]))
 
-    return peakValues
+    return peakValues, hasMultiLayer

avaframe/ana4Stats/getStatsCfg.ini

@@ -20,6 +20,12 @@ varParUnit = m
 nameScenario = releaseScenario
 # restrict type for restricting the result field
 restrictType = pft
+# resultType 1 for plots
+resType1 = pft
+# resultType 2 for plots
+resType2 = pfv
+# if multilayer result files
+layer =
 
 
 ## Uncomment this section FILTER in your local copy of the ini file and add filter parameter and parameter values

avaframe/ana4Stats/probAna.py

@@ -462,7 +462,9 @@ def probAnalysis(avaDir, cfg, modName, parametersDict="", inputDir="", probConf=
 
     # fetch all result files and filter simulations according to parametersDict
     if modName.lower() in ["com1dfa", "com5snowslide", "com6rockavalanche", "com8motpsa", "com9motvoellmy"]:
-        simNameList = cfgHandling.filterSims(avaDir, parametersDict, specDir=inputDir, simDF=simDFActual)
+        simNameList = cfgHandling.filterSims(
+            avaDir, parametersDict, specDir=inputDir, simDF=simDFActual, modName=modName
+        )
         filtering = True
     else:
         simNameList = []
@@ -490,6 +492,18 @@ def probAnalysis(avaDir, cfg, modName, parametersDict="", inputDir="", probConf=
 
         raise FileNotFoundError(message)
 
+    # check layer configuration for multi-layer results
+    peakVar = cfg["GENERAL"]["peakVar"]
+    layer = cfg["GENERAL"].get("layer", "")
+    hasMultiLayer = any(peakFilesDF["layer"] != "")
+    if hasMultiLayer and layer == "":
+        message = (
+            "Multi-layer results detected but no layer specified. "
+            "Set 'layer' in probAnaCfg.ini (e.g. layer = L1)"
+        )
+        log.error(message)
+        raise ValueError(message)
+
     # get header info from peak files - this should be the same for all peakFiles
     header = IOf.readRasterHeader(peakFilesDF["files"][0])
     refData = IOf.readRaster(peakFilesDF["files"][0])
@@ -505,8 +519,9 @@ def probAnalysis(avaDir, cfg, modName, parametersDict="", inputDir="", probConf=
     for m in range(len(peakFilesDF["names"])):
         # only take simulations that match filter criteria from parametersDict
         if (peakFilesDF["simName"][m] in simNameList) or filtering == False:
-            # Load peak field for desired peak field parameter
-            if peakFilesDF["resType"][m] == cfg["GENERAL"]["peakVar"]:
+            # Load peak field for desired peak field parameter, filtering by layer if set
+            layerMatch = (layer == "") or (peakFilesDF["layer"][m].lower() == layer.lower())
+            if peakFilesDF["resType"][m] == peakVar and layerMatch:
                 # Load data
                 fileName = peakFilesDF["files"][m]
                 dataLim = np.zeros((nRows, nCols))
@@ -534,7 +549,7 @@ def probAnalysis(avaDir, cfg, modName, parametersDict="", inputDir="", probConf=
                 )
                 contourDict[fileName.stem] = contourDictXY
 
-                log.info("File Name: %s , simulation parameter %s " % (fileName, cfg["GENERAL"]["peakVar"]))
+                log.info("File Name: %s , simulation parameter %s " % (fileName, peakVar))
 
                 # Check if peak values exceed desired threshold
                 dataLim[data > float(cfg["GENERAL"]["peakLim"])] = 1.0
@@ -543,19 +558,21 @@ def probAnalysis(avaDir, cfg, modName, parametersDict="", inputDir="", probConf=
 
     # Create probability map ranging from 0-1
     probMap = probSum / count
-    unit = pU.cfgPlotUtils["unit%s" % cfg["GENERAL"]["peakVar"]]
+    unit = pU.cfgPlotUtils["unit%s" % peakVar]
+    layerStr = "_%s" % layer if layer else ""
     log.info(
-        "probability analysis performed for peak parameter: %s and a peak value "
-        "threshold of: %s %s" % (cfg["GENERAL"]["peakVar"], cfg["GENERAL"]["peakLim"], unit)
+        "probability analysis performed for peak parameter: %s, layer: %s and a peak value "
+        "threshold of: %s %s" % (peakVar, layer or "single-layer", cfg["GENERAL"]["peakLim"], unit)
     )
     log.info("%s peak fields added to analysis" % count)
 
     # Save to raster file
     avaName = avaDir.name
-    outFileName = "%s_prob_%s_%s_lim%s" % (
+    outFileName = "%s_prob_%s_%s%s_lim%s" % (
         avaName,
         probConf,
-        cfg["GENERAL"]["peakVar"],
+        peakVar,
+        layerStr,
         cfg["GENERAL"]["peakLim"],
     )
     outFile = outDir / outFileName

avaframe/ana4Stats/probAnaCfg.ini

@@ -6,6 +6,8 @@
 [GENERAL]
 # desired result parameter
 peakVar = ppr
+# layer for multi-layer results (e.g. L1, L2), leave empty for single-layer
+layer =
 # unit of desired result parameter
 unit = kPa
 # threshold for probability computations

avaframe/com1DFA/com1DFA.py

@@ -90,6 +90,13 @@ def com1DFAPreprocess(cfgMain, cfgInfo, module=com1DFA):
 
     avalancheDir = cfgMain["MAIN"]["avalancheDir"]
 
+    # batch mode: cfgInfo is a directory containing multiple .ini files
+    if isinstance(cfgInfo, pathlib.Path) and cfgInfo.is_dir():
+        simDict, inputSimFiles, simDFExisting, outDir = com1DFATools.createSimDictFromCfgs(
+            cfgMain, cfgInfo, module=module
+        )
+        return simDict, outDir, inputSimFiles, simDFExisting
+
     # read initial configuration
     if isinstance(cfgInfo, configparser.ConfigParser):
         cfgStart = cfgInfo
@@ -136,13 +143,7 @@ def com1DFAMain(cfgMain, cfgInfo=""):
 
     avalancheDir = cfgMain["MAIN"]["avalancheDir"]
 
-    # Route based on cfgInfo type: directory Path = batch mode, otherwise single config mode
-    if isinstance(cfgInfo, pathlib.Path) and cfgInfo.is_dir():
-        # Batch mode - cfgInfo is directory path (from getModuleConfig with batchCfgDir)
-        simDict, inputSimFiles, simDFExisting, outDir = com1DFATools.createSimDictFromCfgs(cfgMain, cfgInfo)
-    else:
-        # Single config mode - cfgInfo is ConfigParser, file path string, or empty string
-        simDict, outDir, inputSimFiles, simDFExisting = com1DFAPreprocess(cfgMain, cfgInfo)
+    simDict, outDir, inputSimFiles, simDFExisting = com1DFAPreprocess(cfgMain, cfgInfo)
 
     log.info("The following simulations will be performed")
     for key in simDict:

avaframe/com8MoTPSA/com8MoTPSA.py

@@ -35,7 +35,7 @@ def com8MoTPSAMain(cfgMain, cfgInfo=None):
     """
     # Get all necessary information from the configuration files
     currentModule = sys.modules[__name__]
-    simDict, inputSimFiles = mT.MoTGenerateConfigs(cfgMain, cfgInfo, currentModule)
+    simDict, _, inputSimFiles, _ = com1DFA.com1DFAPreprocess(cfgMain, cfgInfo, module=currentModule)
 
     # convert DEM from nan to 0 values
     # TODO: suggest MoT-PSA to handle nan values

avaframe/com8MoTPSA/runAna4ProbAnaCom8MoTPSA.py

@@ -110,7 +110,8 @@ def runProbAna(avalancheDir=''):
                     }
         oP.plotContours(contourDict,
                         cfgProb['GENERAL']['peakVar'],
-                        cfgProb['GENERAL']['peakLim'], pathDict, addLegend=False)
+                        cfgProb['GENERAL']['peakLim'], pathDict, addLegend=False,
+                        layer=cfgProb['GENERAL'].get('layer', ''))
 
     # plot probability maps
     sP.plotProbMap(avalancheDir, inputDir, cfgProb, demPlot=True)

avaframe/com9MoTVoellmy/com9MoTVoellmy.py

@@ -54,7 +54,7 @@ def com9MoTVoellmyMain(cfgMain, cfgInfo=None):
 
     # Get all necessary information from the configuration files
     currentModule = sys.modules[__name__]  # As if you would to import com9MoTVoellmy
-    simDict, inputSimFiles = mT.MoTGenerateConfigs(cfgMain, cfgInfo, currentModule)
+    simDict, _, inputSimFiles, _ = com1DFA.com1DFAPreprocess(cfgMain, cfgInfo, module=currentModule)
 
     # convert DEM from nan to 0 values
     # TODO: suggest MoT-PSA to handle nan values

avaframe/in3Utils/MoTUtils.py

@@ -7,7 +7,6 @@
 
 import numpy as np
 
-from avaframe.com1DFA import com1DFA as com1DFA
 from avaframe.in2Trans import rasterUtils as rU
 
 log = logging.getLogger(__name__)
@@ -124,38 +123,6 @@ def runAndCheckMoT(command):
                 log.info(line)
 
 
-def MoTGenerateConfigs(cfgMain, cfgInfo, currentModule):
-    """
-    Creates configuration objects for com8MoTPSA and com9MoTVoellmy.
-
-    Parameters
-    ------------
-    cfgMain: configparser object
-        main configuration of AvaFrame
-    cfgInfo: str or pathlib Path or configparser object
-        path to configuration file if overwrite is desired - optional
-        if not local (if available) or default configuration will be loaded
-        if cfgInfo is a configparser object take this as initial config
-    currentModule: module object
-        is being passed to cfgUtils.writeCfgFile to create the correct the cfg
-
-
-    Returns
-    --------
-    simDict: dict
-        dictionary with one key per simulation to perform including its config object
-    inputSimFiles: dict
-        dictionary with input files info
-    """
-
-    # preprocessing to create configuration objects for all simulations to run
-    simDict, outDir, inputSimFiles, simDFExisting = com1DFA.com1DFAPreprocess(
-        cfgMain, cfgInfo, module=currentModule
-    )
-
-    return simDict, inputSimFiles
-
-
 def copyMoTFiles(workDir, outputDir, searchString, replaceString):
     """
     Copy and rename MoT result files from work directory to output directory.