[com1; in3] save files during run; enable to resume computation if interrupted

avaframe/com1DFA/com1DFATools.py

@@ -3,6 +3,7 @@
 """
 
 import configparser
+
 # Load modules
 import logging
 import math
@@ -16,6 +17,7 @@
 from avaframe.com1DFA import com1DFA
 from avaframe.in1Data import getInput as gI
 from avaframe.in3Utils import cfgUtils
+from avaframe.in3Utils import fileHandlerUtils as fU
 
 # create local logger
 # change log level in calling module to DEBUG to see log messages
@@ -112,7 +114,7 @@ def compareSimCfgToDefaultCfgCom1DFA(simCfg):
 
     """
 
-    log.info('Comparing simCfg to default cfg')
+    log.info("Comparing simCfg to default cfg")
 
     defaultIdentifierString = "D"
 
@@ -134,9 +136,9 @@ def compareSimCfgToDefaultCfgCom1DFA(simCfg):
     if simCfg["GENERAL"]["simTypeList"] == "ent" and simCfg["GENERAL"]["entThFromShp"] == "True":
         defaultEntTh = defCfg["GENERAL"]["entThIfMissingInShp"]
 
-        if not all([x == defaultEntTh for x in simCfg["INPUT"]["entThThickness"].split('|')]):
+        if not all([x == defaultEntTh for x in simCfg["INPUT"]["entThThickness"].split("|")]):
             defaultIdentifierString = "C"
-            log.info('Non-default entrainment value(s) used: %s' % simCfg["INPUT"]["entThThickness"])
+            log.info("Non-default entrainment value(s) used: %s" % simCfg["INPUT"]["entThThickness"])
 
     # Entrainment might not be set in shpfile, but still the default from
     # ini file is used. This is still default D and not changed C
@@ -169,11 +171,11 @@ def compareSimCfgToDefaultCfgCom1DFA(simCfg):
     valuesChanged = dict()
     # This needs to be checked AFTER check for type_changes
     if "values_changed" in diff:
-        defaultIdentifierString = 'C'
+        defaultIdentifierString = "C"
         for key, val in diff["values_changed"].items():
             valuesChanged[_cleanDiffKey(key)] = val
 
-        log.info('Comparing to default cfg, values changed:')
+        log.info("Comparing to default cfg, values changed:")
         log.info(valuesChanged)
 
     else:
@@ -323,8 +325,11 @@ def initializeInputs(avalancheDir, cleanRemeshedRasters):
     _, outDir = inDirs.initialiseRunDirs(avalancheDir, modName, cleanRemeshedRasters)
 
     # first fetch info on already existing simulations in Outputs
-    # if it is needed to reproduce exactly the hash - need to be strings with exactly the same number of digits!!
-    simDFExisting, simNameExisting = cfgUtils.readAllConfigurationInfo(avalancheDir, specDir="")
+    # if need to reproduce exactly the hash - need to be strings with exactly the same number of digits!!
+    # searchCfgFiles=True enables to search for preformed sims if run has been interrupted
+    simDFExisting, simNameExisting = cfgUtils.readConfigurationInfoFromDone(
+        avalancheDir, specDir='',
+    )
 
     # fetch input data - dem, release-, entrainment- and resistance areas (and secondary release areas)
     inputSimFilesAll = gI.getInputDataCom1DFA(avalancheDir)

avaframe/com1DFA/particleTools.py

@@ -868,7 +868,7 @@ def readPartFromPickle(inDir, simName='', flagAvaDir=False, comModule='com1DFA')
     return Particles, timeStepInfo
 
 
-def savePartToCsv(particleProperties, dictList, outDir):
+def savePartToCsv(particleProperties, dictList, outDir, countParticleCsv=None):
     """ Save each particle dictionary from a list to a csv file;
         works also for one dictionary instead of list
 
@@ -882,6 +882,9 @@ def savePartToCsv(particleProperties, dictList, outDir):
         outDir: str
             path to output directory; particlesCSV will be created in this
             outDir
+        countParticleCsv: int
+            number of particlesDict to be saved according to time step - ranging from 0...n where n is the total number
+            of saved particleDicts - if list of dicts is saved this parameter is ignored
     """
 
     # set output directory
@@ -893,9 +896,20 @@ def savePartToCsv(particleProperties, dictList, outDir):
 
     # write particles locations and properties to csv file
     nParticles = len(dictList)
-    count = 0
+    if nParticles == 1:
+        if countParticleCsv is None:
+            message = 'Indicator for step in timeseries for particlesDict to be saved is required, set countPartCsv'
+            log.error(message)
+            raise AssertionError(message)
+        else:
+            count = countParticleCsv
+    else:
+        count = 0
     for m in range(nParticles):
-        particles = dictList[count]
+        if nParticles == 1:
+            particles = dictList[0]
+        else:
+            particles = dictList[count]
         simName = particles['simName']
         csvData = {}
         csvData['X'] = particles['x'] + particles['xllcenter']

avaframe/in3Utils/cfgUtils.py

@@ -476,7 +476,7 @@ def writeDictToJson(inDict, outFilePath):
     f.close()
 
 
-def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCSV=False, specDir=''):
+def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCSV=False, specDir='', simNameList=[]):
     """ Read configurations from all simulations configuration ini files from directory
 
         Parameters
@@ -489,6 +489,8 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS
             True if configuration dataFrame shall be written to csv file
         specDir: str
             path to a directory where simulation configuration files can be found - optional
+        simNameList: list
+            if non-empty list only use cfgFiles that are included within simNameList
 
         Returns
         --------
@@ -501,7 +503,7 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS
         inDir = pathlib.Path(specDir, 'configurationFiles')
     else:
         inDir = pathlib.Path(avaDir, 'Outputs', comModule, 'configurationFiles')
-    configFiles = inDir.glob('*.ini')
+    configFiles = list(inDir.glob('*.ini'))
 
     if not inDir.is_dir():
         message = 'configuration file directory not found: %s' % (inDir)
@@ -512,34 +514,41 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS
         log.error(message)
         raise FileNotFoundError(message)
 
-    # create confiparser object, convert to json object, write to dataFrame
-    # append all dataFrames
-    simDF = ''
-    for cFile in configFiles:
-        if 'sourceConfiguration' not in str(cFile):
-            simName = pathlib.Path(cFile).stem
-            if '_AF_' in simName:
-                nameParts = simName.split('_AF_')
-                infoParts = nameParts[1].split('_')
-
-            else:
-                nameParts = simName.split('_')
-                infoParts = nameParts[1:]
-            simHash = infoParts[0]
-            cfgObject = readCfgFile(avaDir, fileName=cFile)
-            simDF = appendCgf2DF(simHash, simName, cfgObject, simDF)
-
-    # convert numeric parameters to numerics
-    simDF = convertDF2numerics(simDF)
-
-    # add default configuration
-    if standardCfg != '':
-        # read default configuration of this module
-        simDF = appendCgf2DF('current standard', 'current standard', standardCfg, simDF)
-
-    # if writeCSV, write dataFrame to csv file
-    if writeCSV:
-        writeAllConfigurationInfo(avaDir, simDF, specDir=specDir)
+    # if a simNameList is provided only look for the files with matching simName
+    if simNameList != []:
+        configFiles = [cfgF for cfgF in configFiles if cfgF.stem in simNameList]
+
+    if len(configFiles) == 0:
+        simDF = None
+    else:
+        # create configparser object, convert to json object, write to dataFrame
+        # append all dataFrames
+        simDF = ''
+        for cFile in configFiles:
+            if 'sourceConfiguration' not in str(cFile):
+                simName = pathlib.Path(cFile).stem
+                if '_AF_' in simName:
+                    nameParts = simName.split('_AF_')
+                    infoParts = nameParts[1].split('_')
+
+                else:
+                    nameParts = simName.split('_')
+                    infoParts = nameParts[1:]
+                simHash = infoParts[0]
+                cfgObject = readCfgFile(avaDir, fileName=cFile)
+                simDF = appendCgf2DF(simHash, simName, cfgObject, simDF)
+
+        # convert numeric parameters to numerics
+        simDF = convertDF2numerics(simDF)
+
+        # add default configuration
+        if standardCfg != '':
+            # read default configuration of this module
+            simDF = appendCgf2DF('current standard', 'current standard', standardCfg, simDF)
+
+        # if writeCSV, write dataFrame to csv file
+        if writeCSV:
+            writeAllConfigurationInfo(avaDir, simDF, specDir=specDir)
 
     return simDF
 
@@ -669,6 +678,61 @@ def setStrnanToNan(simDF, simDFTest, name):
             log.info('%s for index: %s set to numpy nan' % (name, index))
     return simDF
 
+def readConfigurationInfoFromDone(avaDir, specDir='', latest=False):
+    """ Check avaName/Outputs/com1DFA/configurationFilesDone and pass
+        names of all files found in this directory and create corresponding simDF
+        this is useful if e.g. no allConfigurations.csv has
+        been written but already some simulations have been performed as a txt file is saved in
+        avaName/Outputs/com1DFA/configurationFiles after the respective simulation has been run
+        whereas the allConfigurations file is written at the end of a call to com1DFAMain that can
+        include several individual sims
+        if latest=True only look for latest simulations in avaName/Outputs/com1DFA/configurationFilesLatest
+
+        Parameters
+        -----------
+        avaDir: str
+            path to avalanche directory
+        specDir: str
+            path to a directory where simulation configuration files directory called configurationFiles can be found - optional
+        latest: bool
+            if True check for files found in avaName/Outputs/com1DFA/configurationFilesLatest
+
+        Returns
+        --------
+        simDF: pandas DataFrame
+            DF with all the simulation configurations
+        simDFName: array
+            simName column of the dataframe
+    """
+
+    # collect all configuration files for this module from directory
+    if specDir != '':
+        inDir = pathlib.Path(specDir, 'configurationFiles')
+    else:
+        inDir = pathlib.Path(avaDir, 'Outputs', 'com1DFA', 'configurationFiles')
+
+    # search inDir/configurationFilesDone or inDir/configurationFilesLatest (depending on latest flag) for already existing sims
+    if latest:
+        configDir = inDir / 'configurationFilesLatest'
+    else:
+        configDir = inDir / 'configurationFilesDone'
+
+    existingSims = list(configDir.glob('*.ini'))
+
+    simNameExisting = []
+    for fName in existingSims:
+        simNameExisting.append(fName.stem)
+    if list((inDir / 'configurationFilesDone').glob('*.ini')) == []:
+        log.info('No existing simulations in Outputs found')
+        simDF = None
+    else:
+        # create simDF (dataFrame with one row per simulation of configuration files found in configDir)
+        simDF = createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCSV=False, specDir=specDir,
+                                        simNameList=simNameExisting)
+
+
+    return simDF, simNameExisting
+
 
 def readAllConfigurationInfo(avaDir, specDir='', configCsvName='allConfigurations'):
     """ Read allConfigurations.csv file as dataFrame from directory

avaframe/in3Utils/initialiseDirs.py

@@ -39,6 +39,8 @@ def initialiseRunDirs(avaDir, modName, cleanRemeshedRasters):
     outputDir = pathlib.Path(avaDir, 'Outputs', modName)
     fU.makeADir(outputDir)
     workDir = pathlib.Path(avaDir, 'Work', modName)
+    configDoneDir = pathlib.Path(avaDir, 'Outputs', modName, 'configurationFiles', 'configurationFilesDone')
+    fU.makeADir(configDoneDir)
     # If Work directory already exists - error
     if workDir.is_dir():
         message = 'Work directory %s already exists - delete first!' % (workDir)
@@ -51,4 +53,7 @@ def initialiseRunDirs(avaDir, modName, cleanRemeshedRasters):
     if cleanRemeshedRasters is True:
         initProj.cleanRemeshedDir(avaDir)
 
+    # first clean configurationFilesLatest dir and create new one
+    initProj.cleanLatestConfigurationsDirAndCreate(avaDir, modName)
+
     return workDir, outputDir

avaframe/in3Utils/initializeProject.py

@@ -9,6 +9,8 @@
 import shutil
 import pathlib
 
+import avaframe.in3Utils.fileHandlerUtils as fU
+
 log = logging.getLogger(__name__)
 
 
@@ -145,6 +147,36 @@ def cleanRemeshedDir(avaDir):
     return 'SUCCESS'
 
 
+def cleanLatestConfigurationsDirAndCreate(avaDir, modName):
+    """ clean configurationFilesLatest folder in avaDir/modName/Outputs/
+
+        Parameters
+        ------------
+        avaDir: str or pathlib patch
+            path to avalanche directory
+        modName: str
+            name of module
+    """
+
+    avaDirOutputs = pathlib.Path(avaDir, 'Outputs', modName, 'configurationFiles')
+    avaDirOutputsString = str(avaDirOutputs)
+
+    # check for empty or non string variable
+    result = _checkAvaDirVariable(avaDirOutputsString)
+    if 'SUCCESS' not in result:
+        return result
+
+    # clean directory
+    folderName = 'configurationFilesLatest'
+    _checkForFolderAndDelete(avaDirOutputsString, folderName)
+    log.info('Cleaned %s/%s directory' % (avaDirOutputsString,folderName))
+
+    # create new dir configurationFilesLatest
+    latestConfigDir = avaDirOutputs / folderName
+    fU.makeADir(latestConfigDir)
+
+    return 'SUCCESS'
+
 def initializeFolderStruct(pathAvaName, removeExisting=False):
     ''' Initialize the standard folder structure. If removeExisting is true,
     deletes any existing folders! BEWARE!

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesDone/relKot_43475422e2_C_S_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesDone/relKot_75696a6497_C_M_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesDone/relKot_7aa27ecfe2_C_M_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesDone/relKot_97de9921c1_C_M_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesDone/relKot_99c96349a9_C_S_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesDone/relKot_abdc3c20af_C_S_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesDone/relKot_bd0086942d_C_S_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesDone/relKot_d8c3f1168f_C_M_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesDone/relKot_f0ffa6b026_C_S_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesLatest/relKot_75696a6497_C_M_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesLatest/relKot_97de9921c1_C_M_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config

avaframe/tests/data/testReadConfig/Outputs/com1DFA/configurationFiles/configurationFilesLatest/relKot_d8c3f1168f_C_M_null_dfa.ini

@@ -0,0 +1 @@
+see directory configurationFiles for info on config