From 8c36410370dd34b234878d9baf0e1ca6b1408d3e Mon Sep 17 00:00:00 2001 From: margrietpalm Date: Wed, 22 May 2024 15:36:05 +0200 Subject: [PATCH] Make gridbuilder compatible to upgrades in threedi-schema 222 (#356) --- CHANGES.rst | 4 +- setup.py | 2 +- threedigrid_builder/__init__.py | 2 +- threedigrid_builder/base/settings.py | 37 +++++++-- threedigrid_builder/interface/db.py | 109 +++++++++++++++++---------- threedigrid_builder/tests/test_db.py | 4 +- 6 files changed, 106 insertions(+), 52 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8d8db85f..fe4b56ba 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,10 +1,10 @@ Changelog of threedigrid-builder ================================ -1.14.3 (unreleased) +1.15.0 (unreleased) ------------------- -- Nothing changed yet. +- Adapt gridbuilder to work with schema upgrades for model settings (0.222) 1.14.2 (2024-05-17) diff --git a/setup.py b/setup.py index 187dd9e3..a213682d 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def get_version(): install_requires = [ "numpy>=1.15", - "threedi-schema==0.220.*", + "threedi-schema==0.222.*", "shapely>=2", "pyproj>=3", "condenser[geo]>=0.1.1", diff --git a/threedigrid_builder/__init__.py b/threedigrid_builder/__init__.py index 374e9073..47d6d26f 100644 --- a/threedigrid_builder/__init__.py +++ b/threedigrid_builder/__init__.py @@ -2,5 +2,5 @@ from .exceptions import * # NOQA # fmt: off -__version__ = '1.14.3.dev0' +__version__ = '1.14.1.dev2' # fmt: on diff --git a/threedigrid_builder/base/settings.py b/threedigrid_builder/base/settings.py index 4b9ce98b..d02aa03d 100644 --- a/threedigrid_builder/base/settings.py +++ b/threedigrid_builder/base/settings.py @@ -1,3 +1,4 @@ +import copy from dataclasses import dataclass from dataclasses import fields from threedigrid_builder.base import is_int_enum @@ -7,7 +8,7 @@ from threedigrid_builder.constants import InitializationType from threedigrid_builder.constants import InterflowType from threedigrid_builder.exceptions import SchematisationError -from typing import Optional +from typing import Any, Dict, Optional import numpy as np @@ -21,6 +22,10 @@ def greater_zero_check(obj, attr): raise SchematisationError(f"'{attr}' must be greater than 0.") +def replace_keys(dict: Dict[str, Any], key_map: Dict[str, str]) -> Dict[str, Any]: + return {key if key not in key_map else key_map[key] : val for key, val in dict.items()} + + @dataclass class GridSettings: """Settings necessary for threedigrid-builder.""" @@ -39,6 +44,10 @@ class GridSettings: @classmethod def from_dict(cls, dct): """Construct skipping unknown fields and None values""" + schema_to_builder_map = {'minimum_cell_size': 'grid_space', + 'calculation_point_distance_1d': 'dist_calc_points', + 'nr_grid_levels': 'kmax'} + dct = replace_keys(copy.copy(dct), schema_to_builder_map) class_fields = {f.name for f in fields(cls)} return cls( **{k: v for k, v in dct.items() if k in class_fields and v is not None} @@ -47,7 +56,7 @@ def from_dict(cls, dct): def __post_init__(self): # validations if self.use_2d: - for field in ["grid_space", "kmax"]: + for field in ["grid_sminimum_cell_sizepace", "nr_grid_levels"]: greater_zero_check(self, field) @@ -55,16 +64,18 @@ def __post_init__(self): class TablesSettings: """Settings necessary for threedi-tables.""" - ## from GlobalSettings + ## from ModelSettings table_step_size: float frict_coef: float frict_coef_type: InitializationType frict_type: FrictionType = FrictionType.MANNING - interception_global: Optional[float] = None - interception_type: Optional[InitializationType] = None table_step_size_1d: float = None # actual default is set in __post_init__ maximum_table_step_size: float = None # actual default is set in __post_init__ + ## From Interception + interception_global: Optional[float] = None + interception_type: Optional[InitializationType] = None + # TODO --> https://github.com/nens/threedigrid-builder/issues/86 manhole_storage_area: Optional[float] = None @@ -150,6 +161,22 @@ def __post_init__(self): def from_dict(cls, dct): """Construct skipping unknown fields and None values""" class_fields = {f.name for f in fields(cls)} + schema_to_builder_map = {"groundwater_hydraulic_conductivity": "groundwater_hydro_connectivity", + "groundwater_hydraulic_conductivity_aggregation": "groundwater_hydro_connectivity_type", + "groundwater_impervious_layer_level_aggregation": "groundwater_impervious_layer_level_type", + "infiltration_decay_period_aggregation": "infiltration_decay_period_type", + "initial_infiltration_rate_aggregation": "initial_infiltration_rate_type", + "phreatic_storage_capacity_aggregation": "phreatic_storage_capacity_type", + "max_infiltration_volume": "max_infiltration_capacity", + "max_infiltration_volume_type": "max_infiltration_capacity_type", + "manhole_aboveground_storage_area": "manhole_storage_area", + "friction_coefficient": "frict_coef", + "minimum_table_step_size": "table_step_size", + "friction_type": "frict_type", + "friction_coefficient_type": "frict_coef_type", + "interception": "interception_global", + } + dct = replace_keys(copy.copy(dct), schema_to_builder_map) return cls( **{k: v for k, v in dct.items() if k in class_fields and v is not None} ) diff --git a/threedigrid_builder/interface/db.py b/threedigrid_builder/interface/db.py index 6e361cb0..1e9f8733 100644 --- a/threedigrid_builder/interface/db.py +++ b/threedigrid_builder/interface/db.py @@ -44,7 +44,7 @@ # hardcoded source projection SOURCE_EPSG = 4326 -MIN_SQLITE_VERSION = 217 +MIN_SQLITE_VERSION = 222 DAY_IN_SECONDS = 24.0 * 3600.0 @@ -82,7 +82,6 @@ def _set_initialization_type( file_field = f"{global_field}_file" if not type_field: type_field = f"{global_field}_type" - # If the ``file_field`` contains a value, the initialization type will be changed to # the ``default_type``, if supplied. if dct[file_field]: @@ -145,56 +144,51 @@ def get_settings(self) -> dict: """ with self.get_session() as session: - global_ = session.query(models.GlobalSetting).order_by("id").first() - if global_.groundwater_settings_id is not None: - groundwater = _object_as_dict( - session.query(models.GroundWater) - .filter_by(id=global_.groundwater_settings_id) - .one() - ) + model_settings = session.query(models.ModelSettings).order_by("id").first() + if ( + model_settings.use_groundwater_flow + or model_settings.use_groundwater_storage + ): + groundwater = _object_as_dict(session.query(models.GroundWater).one()) else: groundwater = {} - if global_.interflow_settings_id is not None: - interflow = _object_as_dict( - session.query(models.Interflow) - .filter_by(id=global_.interflow_settings_id) - .one() - ) + if model_settings.use_interflow: + interflow = _object_as_dict(session.query(models.Interflow).one()) else: interflow = {} - if global_.simple_infiltration_settings_id is not None: + if model_settings.use_simple_infiltration: infiltration = _object_as_dict( - session.query(models.SimpleInfiltration) - .filter_by(id=global_.simple_infiltration_settings_id) - .one() + session.query(models.SimpleInfiltration).one() ) - # older sqlites have no max_infiltration_capacity field - infiltration.setdefault("max_infiltration_capacity", None) else: infiltration = {} - if global_.vegetation_drag_settings_id is not None: + if model_settings.use_vegetation_drag_2d: vegetation_drag = _object_as_dict( - session.query(models.VegetationDrag) - .filter_by(id=global_.vegetation_drag_settings_id) - .one(), + session.query(models.VegetationDrag).one() ) else: vegetation_drag = {} - global_ = _object_as_dict(global_) + if model_settings.use_interception: + interception = _object_as_dict(session.query(models.Interception).one()) + else: + interception = {} + model_settings = _object_as_dict(model_settings) # record if there is a DEM file to be expected # Note: use_2d_flow only determines whether there are flow lines - global_["use_2d"] = bool(global_["dem_file"]) + model_settings["use_2d"] = bool(model_settings["dem_file"]) # set/adapt initialization types to include information about file presence NO_AGG = InitializationType.NO_AGG AVERAGE = InitializationType.AVERAGE _set_initialization_type( - global_, "frict_coef", default=AVERAGE if global_["frict_avg"] else NO_AGG + model_settings, + "friction_coefficient", + default=AVERAGE if model_settings["friction_averaging"] else NO_AGG, ) _set_initialization_type( - global_, - "interception_global", + interception, + "interception", file_field="interception_file", type_field="interception_type", default=NO_AGG, @@ -207,17 +201,37 @@ def get_settings(self) -> dict: if infiltration: _set_initialization_type(infiltration, "infiltration_rate", default=NO_AGG) _set_initialization_type( - infiltration, "max_infiltration_capacity", default=NO_AGG + infiltration, "max_infiltration_volume", default=NO_AGG ) - if groundwater: # default is what the user supplied (MIN/MAX/AVERAGE) - _set_initialization_type(groundwater, "groundwater_impervious_layer_level") - _set_initialization_type(groundwater, "phreatic_storage_capacity") + + _set_initialization_type( + groundwater, + "groundwater_impervious_layer_level", + type_field="groundwater_impervious_layer_level_aggregation", + ) + _set_initialization_type( + groundwater, + "phreatic_storage_capacity", + type_field="phreatic_storage_capacity_aggregation", + ) _set_initialization_type(groundwater, "equilibrium_infiltration_rate") - _set_initialization_type(groundwater, "initial_infiltration_rate") - _set_initialization_type(groundwater, "infiltration_decay_period") - _set_initialization_type(groundwater, "groundwater_hydro_connectivity") + _set_initialization_type( + groundwater, + "initial_infiltration_rate", + type_field="initial_infiltration_rate_aggregation", + ) + _set_initialization_type( + groundwater, + "infiltration_decay_period", + type_field="infiltration_decay_period_aggregation", + ) + _set_initialization_type( + groundwater, + "groundwater_hydraulic_conductivity", + type_field="groundwater_hydraulic_conductivity_aggregation", + ) if vegetation_drag: _set_initialization_type( @@ -232,14 +246,27 @@ def get_settings(self) -> dict: _set_initialization_type( vegetation_drag, "vegetation_drag_coefficient", default=NO_AGG ) + # Copy Simulation Template Settings to model_settings dict + template_settings = _object_as_dict( + session.query(models.SimulationTemplateSettings).order_by("id").first() + ) + model_settings["name"] = template_settings["name"] + model_settings["use_0d_inflow"] = template_settings["use_0d_inflow"] - grid_settings = GridSettings.from_dict(global_) + grid_settings = GridSettings.from_dict(model_settings) tables_settings = TablesSettings.from_dict( - {**groundwater, **interflow, **infiltration, **vegetation_drag, **global_} + { + **groundwater, + **interflow, + **infiltration, + **vegetation_drag, + **model_settings, + **interception, + } ) return { - "epsg_code": global_["epsg_code"], - "model_name": global_["name"], + "epsg_code": model_settings["epsg_code"], + "model_name": model_settings["name"], "grid_settings": grid_settings, "tables_settings": tables_settings, } diff --git a/threedigrid_builder/tests/test_db.py b/threedigrid_builder/tests/test_db.py index cb1439f9..88ede2a0 100644 --- a/threedigrid_builder/tests/test_db.py +++ b/threedigrid_builder/tests/test_db.py @@ -37,7 +37,7 @@ def test_init(tmp_path): with mock.patch( "threedigrid_builder.interface.db.ThreediDatabase" ) as db, mock.patch.object(SQLite, "get_version") as get_version: - get_version.return_value = 217 + get_version.return_value = 222 sqlite = SQLite(path) db.assert_called_with(path) @@ -65,7 +65,7 @@ def test_init_bad_version(tmp_path): def test_get_version(db): - assert db.get_version() == 220 + assert db.get_version() == 222 def test_get_boundary_conditions_1d(db):