From ac88e76c8a35057e4eb42d982a823a42bfb7880c Mon Sep 17 00:00:00 2001 From: Felix Oesterle <6945681+fso42@users.noreply.github.com> Date: Wed, 28 Jan 2026 09:15:48 +0100 Subject: [PATCH] refactor(core): Adjust MacOS related imports and pixi setup - Added macOS `arm64` support in `pyproject.toml` platforms configuration. fix(pyproject): uncomment `numpy` dependency for `qgis` feature --- avaframe/com1DFA/com1DFA.py | 2 - avaframe/com4FlowPy/flowCore.py | 157 +++++++++++++--------- avaframe/com8MoTPSA/com8MoTPSA.py | 2 - avaframe/com9MoTVoellmy/com9MoTVoellmy.py | 2 - pyproject.toml | 2 +- 5 files changed, 97 insertions(+), 68 deletions(-) diff --git a/avaframe/com1DFA/com1DFA.py b/avaframe/com1DFA/com1DFA.py index 05f872c6f..20fd1b99d 100644 --- a/avaframe/com1DFA/com1DFA.py +++ b/avaframe/com1DFA/com1DFA.py @@ -24,8 +24,6 @@ if os.name == "nt": from multiprocessing.pool import ThreadPool as Pool -elif platform.system() == "Darwin": - from multiprocessing.pool import ThreadPool as Pool else: from multiprocessing import Pool diff --git a/avaframe/com4FlowPy/flowCore.py b/avaframe/com4FlowPy/flowCore.py index a9a2315f7..6af71506b 100644 --- a/avaframe/com4FlowPy/flowCore.py +++ b/avaframe/com4FlowPy/flowCore.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ - Calculation functions (raster level) +Calculation functions (raster level) """ import sys @@ -16,8 +16,6 @@ if os.name == "nt": from multiprocessing.pool import Pool as Pool -elif platform.system() == "Darwin": - from multiprocessing.pool import ThreadPool as Pool else: from multiprocessing import Pool @@ -47,7 +45,9 @@ def get_start_idx(dem, release): altitude_list = [] for i in range(len(row_list)): altitude_list.append(dem[row_list[i], col_list[i]]) - altitude_list, row_list, col_list = list(zip(*sorted(zip(altitude_list, row_list, col_list), reverse=True))) + altitude_list, row_list, col_list = list( + zip(*sorted(zip(altitude_list, row_list, col_list), reverse=True)) + ) # Sort this lists by altitude return row_list, col_list @@ -182,9 +182,9 @@ def run(optTuple): if varUmaxBool: varUmaxArray = np.load(tempDir / ("varUmax_%s_%s.npy" % (optTuple[0], optTuple[1]))) - if optTuple[2]["varUmaxType"].lower() == 'umax': + if optTuple[2]["varUmaxType"].lower() == "umax": varUmaxArray[varUmaxArray > 0] = varUmaxArray[varUmaxArray > 0] ** 2 / 2 / 9.81 - elif optTuple[2]["varUmaxType"].lower() != 'zdeltalim': + elif optTuple[2]["varUmaxType"].lower() != "zdeltalim": log.error("PLease provide the type of the uMax Limit: 'uMax' (in m/s) or zDeltaMax (in m)!") else: varUmaxArray = None @@ -200,18 +200,18 @@ def run(optTuple): varExponentArray = None varParams = { - 'varUmaxBool': varUmaxBool, - 'varUmaxArray': varUmaxArray, - 'varAlphaBool': varAlphaBool, - 'varAlphaArray': varAlphaArray, - 'varExponentBool': varExponentBool, - 'varExponentArray': varExponentArray, + "varUmaxBool": varUmaxBool, + "varUmaxArray": varUmaxArray, + "varAlphaBool": varAlphaBool, + "varAlphaArray": varAlphaArray, + "varExponentBool": varExponentBool, + "varExponentArray": varExponentArray, } # convert release areas to binary (0: no release areas, 1: release areas) # every positive value >0 is interpreted as release area release[release < 0] = 0 - release[release == nodata] = 0 # added in case nodata is non-negative + release[release == nodata] = 0 # added in case nodata is non-negative release[release > 0] = 1 nRel = np.sum(release) @@ -226,7 +226,10 @@ def run(optTuple): ) release_list = split_release(release, nChunks) - log.info("Multiprocessing starts, used Cores/Processes/Chunks: %i/%i/%i" % (MPOptions["nCPU"], nProcesses, nChunks)) + log.info( + "Multiprocessing starts, used Cores/Processes/Chunks: %i/%i/%i" + % (MPOptions["nCPU"], nProcesses, nChunks) + ) with Pool(processes=nProcesses) as pool: results = pool.map( @@ -339,9 +342,11 @@ def run(optTuple): np.maximum(travelLengthMinArray, travelLengthMinList[i]), ) if forestInteraction: - forestIntArray = np.where((forestIntArray >= 0) & (forestIntList[i] >= 0), - np.minimum(forestIntArray, forestIntList[i]), - np.maximum(forestIntArray, forestIntList[i])) + forestIntArray = np.where( + (forestIntArray >= 0) & (forestIntList[i] >= 0), + np.minimum(forestIntArray, forestIntList[i]), + np.maximum(forestIntArray, forestIntList[i]), + ) # Save Calculated tiles np.save(tempDir / ("res_z_delta_%s_%s" % (optTuple[0], optTuple[1])), zDeltaArray) @@ -446,12 +451,12 @@ def updateInfraDirGraph(row, col, parentRow=None, parentCol=None): cellsize = args[8] infraBool = args[9] forestBool = args[10] - varUmaxBool = args[11]['varUmaxBool'] - varUmaxArray = args[11]['varUmaxArray'] - varAlphaBool = args[11]['varAlphaBool'] - varAlphaArray = args[11]['varAlphaArray'] - varExponentBool = args[11]['varExponentBool'] - varExponentArray = args[11]['varExponentArray'] + varUmaxBool = args[11]["varUmaxBool"] + varUmaxArray = args[11]["varUmaxArray"] + varAlphaBool = args[11]["varAlphaBool"] + varAlphaArray = args[11]["varAlphaArray"] + varExponentBool = args[11]["varExponentBool"] + varExponentArray = args[11]["varExponentArray"] fluxDistOldVersionBool = args[12] previewMode = args[13] outputs = args[16] @@ -502,15 +507,15 @@ def updateInfraDirGraph(row, col, parentRow=None, parentCol=None): if infraBool: # if infraBool - here we initialize a directed graph structure - pathTopology = {} # topology of path as directed graph - infraValues = {} # values + pathTopology = {} # topology of path as directed graph + infraValues = {} # values processedCells = {} # dictionary of cells that have been processed already zDeltaPathArray = np.zeros_like(dem, dtype=np.float32) cell_list = [] row_idx = row_list[startcell_idx] col_idx = col_list[startcell_idx] - dem_ng = dem[row_idx - 1: row_idx + 2, col_idx - 1: col_idx + 2] # neighbourhood DEM + dem_ng = dem[row_idx - 1 : row_idx + 2, col_idx - 1 : col_idx + 2] # neighbourhood DEM if varUmaxBool and varUmaxArray is not None: if varUmaxArray[row_idx, col_idx] > 0 and varUmaxArray[row_idx, col_idx] <= 8848: max_z_delta = varUmaxArray[row_idx, col_idx] @@ -526,11 +531,19 @@ def updateInfraDirGraph(row, col, parentRow=None, parentCol=None): continue startcell = Cell( - row_idx, col_idx, - dem_ng, cellsize, - 1, 0, None, - alpha, exp, flux_threshold, max_z_delta, - startcell=True, fluxDistOldVersionBool=fluxDistOldVersionBool, + row_idx, + col_idx, + dem_ng, + cellsize, + 1, + 0, + None, + alpha, + exp, + flux_threshold, + max_z_delta, + startcell=True, + fluxDistOldVersionBool=fluxDistOldVersionBool, FSI=forestArray[row_idx, col_idx] if isinstance(forestArray, np.ndarray) else None, forestParams=forestParams, ) @@ -551,13 +564,13 @@ def updateInfraDirGraph(row, col, parentRow=None, parentCol=None): # number of child cells (successors) to currently processed cell row, col, flux, z_delta = cell.calc_distribution() - if len(flux) > 0: #i.e. if there are child cells + if len(flux) > 0: # i.e. if there are child cells # Sort this lists by z_delta, to start with the highest cell z_delta, flux, row, col = list(zip(*sorted(zip(z_delta, flux, row, col), reverse=False))) if infraBool: # if the current cell is not already in the dir-graph, then we add it here - updateInfraDirGraph(cell.rowindex, cell.colindex) + updateInfraDirGraph(cell.rowindex, cell.colindex) # check if child cells already exist for i in range(idx, len(cell_list)): @@ -580,7 +593,7 @@ def updateInfraDirGraph(row, col, parentRow=None, parentCol=None): k += 1 for k in range(len(row)): - dem_ng = dem[row[k] - 1: row[k] + 2, col[k] - 1: col[k] + 2] # neighbourhood DEM + dem_ng = dem[row[k] - 1 : row[k] + 2, col[k] - 1 : col[k] + 2] # neighbourhood DEM # This bit handles edge cases and noData-values in the DEM!! this is an important piece of code, since # no-data handling is expected (by some users/applications) to behave like here: @@ -599,22 +612,35 @@ def updateInfraDirGraph(row, col, parentRow=None, parentCol=None): else: processedCells[(row[k], col[k])] = 1 - cell_list.append(Cell( - row[k], col[k], - dem_ng, cellsize, - flux[k], z_delta[k], - cell, - alpha, exp, flux_threshold, max_z_delta, - startcell, fluxDistOldVersionBool=fluxDistOldVersionBool, - FSI=forestArray[row[k], col[k]] if isinstance(forestArray, np.ndarray) else None, - forestParams=forestParams, - )) - - zDeltaArray[cell.rowindex, cell.colindex] = max(zDeltaArray[cell.rowindex, cell.colindex], cell.z_delta) + cell_list.append( + Cell( + row[k], + col[k], + dem_ng, + cellsize, + flux[k], + z_delta[k], + cell, + alpha, + exp, + flux_threshold, + max_z_delta, + startcell, + fluxDistOldVersionBool=fluxDistOldVersionBool, + FSI=forestArray[row[k], col[k]] if isinstance(forestArray, np.ndarray) else None, + forestParams=forestParams, + ) + ) + + zDeltaArray[cell.rowindex, cell.colindex] = max( + zDeltaArray[cell.rowindex, cell.colindex], cell.z_delta + ) fluxArray[cell.rowindex, cell.colindex] = max(fluxArray[cell.rowindex, cell.colindex], cell.flux) routFluxSumArray[cell.rowindex, cell.colindex] += cell.flux depFluxSumArray[cell.rowindex, cell.colindex] += cell.fluxDep - zDeltaPathArray[cell.rowindex, cell.colindex] = max(zDeltaPathArray[cell.rowindex, cell.colindex], cell.z_delta) + zDeltaPathArray[cell.rowindex, cell.colindex] = max( + zDeltaPathArray[cell.rowindex, cell.colindex], cell.z_delta + ) if "fpTravelAngleMax" in outputs or "fpTravelAngle" in outputs: fpTravelAngleMaxArray[cell.rowindex, cell.colindex] = max( fpTravelAngleMaxArray[cell.rowindex, cell.colindex], cell.max_gamma @@ -628,8 +654,9 @@ def updateInfraDirGraph(row, col, parentRow=None, parentCol=None): fpTravelAngleMinArray[cell.rowindex, cell.colindex] = max( fpTravelAngleMinArray[cell.rowindex, cell.colindex], cell.max_gamma ) - slTravelAngleArray[cell.rowindex, cell.colindex] = max(slTravelAngleArray[cell.rowindex, cell.colindex], - cell.sl_gamma) + slTravelAngleArray[cell.rowindex, cell.colindex] = max( + slTravelAngleArray[cell.rowindex, cell.colindex], cell.sl_gamma + ) if "travelLengthMax" in outputs or "travelLength" in outputs: travelLengthMaxArray[cell.rowindex, cell.colindex] = max( travelLengthMaxArray[cell.rowindex, cell.colindex], cell.min_distance @@ -648,20 +675,26 @@ def updateInfraDirGraph(row, col, parentRow=None, parentCol=None): if forestInteraction: if forestIntArray[cell.rowindex, cell.colindex] >= 0 and cell.forestIntCount >= 0: - forestIntArray[cell.rowindex, cell.colindex] = min(forestIntArray[cell.rowindex, cell.colindex], - cell.forestIntCount) + forestIntArray[cell.rowindex, cell.colindex] = min( + forestIntArray[cell.rowindex, cell.colindex], cell.forestIntCount + ) else: - forestIntArray[cell.rowindex, cell.colindex] = max(forestIntArray[cell.rowindex, cell.colindex], - cell.forestIntCount) + forestIntArray[cell.rowindex, cell.colindex] = max( + forestIntArray[cell.rowindex, cell.colindex], cell.forestIntCount + ) if infraBool: # if 'infraBool' is True - i.e. calculation is performed with infrastructure information # then we perform the back-tracking of the stored directed graph (topology and node values) - updatedInfraValues = backTracking(pathTopology, infraValues) # actual "back-tracking" for current process-path + updatedInfraValues = backTracking( + pathTopology, infraValues + ) # actual "back-tracking" for current process-path for key, val in updatedInfraValues.items(): - backcalc[key[0], key[1]] = max(backcalc[key[0], key[1]], val) # writing max-values to back-tracking array + backcalc[key[0], key[1]] = max( + backcalc[key[0], key[1]], val + ) # writing max-values to back-tracking array del pathTopology, infraValues, updatedInfraValues gc.collect() @@ -822,6 +855,7 @@ def handleMemoryAvailability(recheckInterval=30): while not enoughMemoryAvailable(): time.sleep(recheckInterval) + def backTracking(topologyDict, infraValDict): """ peform the back-tracking of infrastructure values across the dir-graph @@ -834,9 +868,9 @@ def backTracking(topologyDict, infraValDict): where parent nodes (colindex, rowindex) serve as keys and children of the respective parent node are stored as list items for the respective key infraValDict : dict - dictionay containing information if a node is an infrastructure cell + dictionay containing information if a node is an infrastructure cell (value at key 'node' > 0) or not (value at key 'node' == 0) - + Returns ----------- infraValDict : dict @@ -845,7 +879,7 @@ def backTracking(topologyDict, infraValDict): """ # sort valDict (so we start traversing from highest infrastructure cells first) # this makes the algorithm more efficient - valDictSorted = {k: v for k, v in sorted(infraValDict.items(), key= lambda item: item[1], reverse=True)} + valDictSorted = {k: v for k, v in sorted(infraValDict.items(), key=lambda item: item[1], reverse=True)} # reverse the graph topology, so "parents" become "children" reverseGraph = reverseTopology(topologyDict) @@ -873,8 +907,9 @@ def propagateInfraVal(node, infraValToPropagate, visited): return infraValDict + def reverseTopology(topologyDict): - ''' + """ reverse graph topology i.e. directions of graph edges connecting the nodes in the dir graph @@ -884,12 +919,12 @@ def reverseTopology(topologyDict): dictionary containing the topology of the modeled process path where parent nodes (colindex, rowindex) serve as keys and children of the respective parent node are stored as list items for the respective key - + Returns ----------- reverseGraph : dict dir-graph with reversed edges in same dictionary format as orignial input - ''' + """ reverseGraph = {} for parentNode, childNodes in topologyDict.items(): diff --git a/avaframe/com8MoTPSA/com8MoTPSA.py b/avaframe/com8MoTPSA/com8MoTPSA.py index a9bbecc86..9f2b63b6e 100644 --- a/avaframe/com8MoTPSA/com8MoTPSA.py +++ b/avaframe/com8MoTPSA/com8MoTPSA.py @@ -11,8 +11,6 @@ if os.name == "nt": from multiprocessing.pool import ThreadPool as Pool -elif platform.system() == "Darwin": - from multiprocessing.pool import ThreadPool as Pool else: from multiprocessing import Pool diff --git a/avaframe/com9MoTVoellmy/com9MoTVoellmy.py b/avaframe/com9MoTVoellmy/com9MoTVoellmy.py index 82cf5d9fa..ce31b82c1 100644 --- a/avaframe/com9MoTVoellmy/com9MoTVoellmy.py +++ b/avaframe/com9MoTVoellmy/com9MoTVoellmy.py @@ -7,8 +7,6 @@ if os.name == "nt": from multiprocessing.pool import ThreadPool as Pool -elif platform.system() == "Darwin": - from multiprocessing.pool import ThreadPool as Pool else: from multiprocessing import Pool diff --git a/pyproject.toml b/pyproject.toml index 5b66aacc3..afe62dd2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ line-length = 109 # PIXI setup [tool.pixi.workspace] channels = ["https://prefix.dev/conda-forge"] -platforms = ["linux-64", "win-64", "osx-64"] +platforms = ["linux-64", "win-64", "osx-64", "osx-arm64"] [tool.pixi.dependencies] python = ">=3.9,<3.14"