Make gridbuilder compatible to schema upgrades

setup.py

@@ -53,7 +53,7 @@ def get_version():
 
 install_requires = [
     "numpy>=1.15",
-    "threedi-schema==0.220.*",
+    "threedi-schema@https://github.com/nens/threedi-schema/archive/refs/heads/margriet_46_schema_300.zip",
     "shapely>=2",
     "pyproj>=3",
     "condenser[geo]>=0.1.1",

threedigrid_builder/base/settings.py

@@ -1,3 +1,4 @@
+import copy
 from dataclasses import dataclass
 from dataclasses import fields
 from threedigrid_builder.base import is_int_enum
@@ -7,7 +8,7 @@
 from threedigrid_builder.constants import InitializationType
 from threedigrid_builder.constants import InterflowType
 from threedigrid_builder.exceptions import SchematisationError
-from typing import Optional
+from typing import Any, Dict, Optional
 
 import numpy as np
 
@@ -21,6 +22,10 @@ def greater_zero_check(obj, attr):
         raise SchematisationError(f"'{attr}' must be greater than 0.")
 
 
+def replace_keys(dict: Dict[str, Any], key_map: Dict[str, str]) -> Dict[str, Any]:
+    return {key if key not in key_map else key_map[key] : val for key, val in dict.items()}
+
+
 @dataclass
 class GridSettings:
     """Settings necessary for threedigrid-builder."""
@@ -39,6 +44,10 @@ class GridSettings:
     @classmethod
     def from_dict(cls, dct):
         """Construct skipping unknown fields and None values"""
+        schema_to_builder_map = {'minimum_cell_size': 'grid_space',
+                                 'calculation_point_distance_1d': 'dist_calc_points',
+                                 'nr_grid_levels': 'kmax'}
+        dct = replace_keys(copy.copy(dct), schema_to_builder_map)
         class_fields = {f.name for f in fields(cls)}
         return cls(
             **{k: v for k, v in dct.items() if k in class_fields and v is not None}
@@ -47,7 +56,7 @@ def from_dict(cls, dct):
     def __post_init__(self):
         # validations
         if self.use_2d:
-            for field in ["grid_space", "kmax"]:
+            for field in ["grid_sminimum_cell_sizepace", "nr_grid_levels"]:
                 greater_zero_check(self, field)
 
 
@@ -56,6 +65,7 @@ class TablesSettings:
     """Settings necessary for threedi-tables."""
 
     ## from GlobalSettings
+    # TODO: update comments to reflect origins
     table_step_size: float
     frict_coef: float
     frict_coef_type: InitializationType
@@ -150,6 +160,22 @@ def __post_init__(self):
     def from_dict(cls, dct):
         """Construct skipping unknown fields and None values"""
         class_fields = {f.name for f in fields(cls)}
+        schema_to_builder_map = {"groundwater_hydraulic_conductivity": "groundwater_hydro_connectivity",
+                                 "groundwater_hydraulic_conductivity_aggregation": "groundwater_hydro_connectivity_type",
+                                 "groundwater_impervious_layer_level_aggregation": "groundwater_impervious_layer_level_type",
+                                 "infiltration_decay_period_aggregation": "infiltration_decay_period_type",
+                                 "initial_infiltration_rate_aggregation": "initial_infiltration_rate_type",
+                                 "phreatic_storage_capacity_aggregation": "phreatic_storage_capacity_type",
+                                 "max_infiltration_volume": "max_infiltration_capacity",
+                                 "max_infiltration_volume_type": "max_infiltration_capacity_type",
+                                 "manhole_aboveground_storage_area": "manhole_storage_area",
+                                 "friction_coefficient": "frict_coef",
+                                 "minimum_table_step_size": "table_step_size",
+                                 "friction_type": "frict_type",
+                                 "friction_coefficient_type": "frict_coef_type",
+                                 "interception": "interception_global",
+                                 }
+        dct = replace_keys(copy.copy(dct), schema_to_builder_map)
         return cls(
             **{k: v for k, v in dct.items() if k in class_fields and v is not None}
         )

threedigrid_builder/interface/db.py

@@ -44,7 +44,7 @@
 # hardcoded source projection
 SOURCE_EPSG = 4326
 
-MIN_SQLITE_VERSION = 217
+MIN_SQLITE_VERSION = 300
 
 DAY_IN_SECONDS = 24.0 * 3600.0
 
@@ -82,7 +82,6 @@ def _set_initialization_type(
         file_field = f"{global_field}_file"
     if not type_field:
         type_field = f"{global_field}_type"
-
     # If the ``file_field`` contains a value, the initialization type will be changed to
     # the ``default_type``, if supplied.
     if dct[file_field]:
@@ -145,56 +144,53 @@ def get_settings(self) -> dict:
         """
 
         with self.get_session() as session:
-            global_ = session.query(models.GlobalSetting).order_by("id").first()
-            if global_.groundwater_settings_id is not None:
-                groundwater = _object_as_dict(
-                    session.query(models.GroundWater)
-                    .filter_by(id=global_.groundwater_settings_id)
-                    .one()
-                )
+            model_settings = session.query(models.ModelSettings).order_by("id").first()
+            if (
+                model_settings.use_groundwater_flow
+                or model_settings.use_groundwater_storage
+            ):
+                groundwater = _object_as_dict(session.query(models.GroundWater).one())
             else:
                 groundwater = {}
-            if global_.interflow_settings_id is not None:
-                interflow = _object_as_dict(
-                    session.query(models.Interflow)
-                    .filter_by(id=global_.interflow_settings_id)
-                    .one()
-                )
+            if model_settings.use_interflow:
+                interflow = _object_as_dict(session.query(models.Interflow).one())
             else:
                 interflow = {}
-            if global_.simple_infiltration_settings_id is not None:
+            if model_settings.use_simple_infiltration:
                 infiltration = _object_as_dict(
-                    session.query(models.SimpleInfiltration)
-                    .filter_by(id=global_.simple_infiltration_settings_id)
-                    .one()
+                    session.query(models.SimpleInfiltration).one()
                 )
                 # older sqlites have no max_infiltration_capacity field
-                infiltration.setdefault("max_infiltration_capacity", None)
+                infiltration.setdefault("max_infiltration_volume", None)
             else:
                 infiltration = {}
-            if global_.vegetation_drag_settings_id is not None:
+            if model_settings.use_vegetation_drag_2d:
                 vegetation_drag = _object_as_dict(
-                    session.query(models.VegetationDrag)
-                    .filter_by(id=global_.vegetation_drag_settings_id)
-                    .one(),
+                    session.query(models.VegetationDrag).one()
                 )
             else:
                 vegetation_drag = {}
-            global_ = _object_as_dict(global_)
+            if model_settings.use_interception:
+                interception = _object_as_dict(session.query(models.Interception).one())
+            else:
+                interception = {}
+            model_settings = _object_as_dict(model_settings)
 
         # record if there is a DEM file to be expected
         # Note: use_2d_flow only determines whether there are flow lines
-        global_["use_2d"] = bool(global_["dem_file"])
+        model_settings["use_2d"] = bool(model_settings["dem_file"])
 
         # set/adapt initialization types to include information about file presence
         NO_AGG = InitializationType.NO_AGG
         AVERAGE = InitializationType.AVERAGE
         _set_initialization_type(
-            global_, "frict_coef", default=AVERAGE if global_["frict_avg"] else NO_AGG
+            model_settings,
+            "friction_coefficient",
+            default=AVERAGE if model_settings["friction_averaging"] else NO_AGG,
         )
         _set_initialization_type(
-            global_,
-            "interception_global",
+            interception,
+            "interception",
             file_field="interception_file",
             type_field="interception_type",
             default=NO_AGG,
@@ -207,17 +203,37 @@ def get_settings(self) -> dict:
         if infiltration:
             _set_initialization_type(infiltration, "infiltration_rate", default=NO_AGG)
             _set_initialization_type(
-                infiltration, "max_infiltration_capacity", default=NO_AGG
+                infiltration, "max_infiltration_volume", default=NO_AGG
             )
-
         if groundwater:
             # default is what the user supplied (MIN/MAX/AVERAGE)
-            _set_initialization_type(groundwater, "groundwater_impervious_layer_level")
-            _set_initialization_type(groundwater, "phreatic_storage_capacity")
+
+            _set_initialization_type(
+                groundwater,
+                "groundwater_impervious_layer_level",
+                type_field="groundwater_impervious_layer_level_aggregation",
+            )
+            _set_initialization_type(
+                groundwater,
+                "phreatic_storage_capacity",
+                type_field="phreatic_storage_capacity_aggregation",
+            )
             _set_initialization_type(groundwater, "equilibrium_infiltration_rate")
-            _set_initialization_type(groundwater, "initial_infiltration_rate")
-            _set_initialization_type(groundwater, "infiltration_decay_period")
-            _set_initialization_type(groundwater, "groundwater_hydro_connectivity")
+            _set_initialization_type(
+                groundwater,
+                "initial_infiltration_rate",
+                type_field="initial_infiltration_rate_aggregation",
+            )
+            _set_initialization_type(
+                groundwater,
+                "infiltration_decay_period",
+                type_field="infiltration_decay_period_aggregation",
+            )
+            _set_initialization_type(
+                groundwater,
+                "groundwater_hydraulic_conductivity",
+                type_field="groundwater_hydraulic_conductivity_aggregation",
+            )
 
         if vegetation_drag:
             _set_initialization_type(
@@ -232,14 +248,27 @@ def get_settings(self) -> dict:
             _set_initialization_type(
                 vegetation_drag, "vegetation_drag_coefficient", default=NO_AGG
             )
+        # Copy Simulation Template Settings to model_settings dict
+        template_settings = _object_as_dict(
+            session.query(models.SimulationTemplateSettings).order_by("id").first()
+        )
+        model_settings["name"] = template_settings["name"]
+        model_settings["use_0d_inflow"] = template_settings["use_0d_inflow"]
 
-        grid_settings = GridSettings.from_dict(global_)
+        grid_settings = GridSettings.from_dict(model_settings)
         tables_settings = TablesSettings.from_dict(
-            {**groundwater, **interflow, **infiltration, **vegetation_drag, **global_}
+            {
+                **groundwater,
+                **interflow,
+                **infiltration,
+                **vegetation_drag,
+                **model_settings,
+                **interception,
+            }
         )
         return {
-            "epsg_code": global_["epsg_code"],
-            "model_name": global_["name"],
+            "epsg_code": model_settings["epsg_code"],
+            "model_name": model_settings["name"],
             "grid_settings": grid_settings,
             "tables_settings": tables_settings,
         }

threedigrid_builder/tests/test_db.py

@@ -37,7 +37,7 @@ def test_init(tmp_path):
     with mock.patch(
         "threedigrid_builder.interface.db.ThreediDatabase"
     ) as db, mock.patch.object(SQLite, "get_version") as get_version:
-        get_version.return_value = 217
+        get_version.return_value = 300
         sqlite = SQLite(path)
 
     db.assert_called_with(path)
@@ -65,7 +65,7 @@ def test_init_bad_version(tmp_path):
 
 
 def test_get_version(db):
-    assert db.get_version() == 220
+    assert db.get_version() == 300
 
 
 def test_get_boundary_conditions_1d(db):