Inflow schema 300 (#366)

Update to work with schema 223

.github/workflows/release.yml

@@ -40,7 +40,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-11]
+        os: [ubuntu-20.04, windows-2019, macos-12]
 
     steps:
       - name: Checkout source

CHANGES.rst

@@ -5,6 +5,7 @@ Changelog of threedigrid-builder
 1.15.1 (unreleased)
 -------------------
 
+- Adapt grid-builder to work with schema upgrades for inflow (0.223)
 - Create quarter administration.
 - Add support for NumPy 2.
 

setup.py

@@ -53,7 +53,7 @@ def get_version():
 
 install_requires = [
     "numpy>=1.15,<3.0",
-    "threedi-schema==0.222.*",
+    "threedi-schema==0.223.*",
     "shapely>=2",
     "pyproj>=3",
     "condenser[geo]>=0.1.1",

threedigrid_builder/application.py

@@ -15,7 +15,6 @@
 from threedigrid_builder.constants import InflowType
 from threedigrid_builder.exceptions import SchematisationError
 from threedigrid_builder.grid import Grid, QuadTree
-from threedigrid_builder.grid.zero_d import ImperviousSurfaces
 from threedigrid_builder.interface import (
     DictOut,
     GDALInterface,
@@ -210,10 +209,7 @@ def _make_gridadmin(
         # process zero-d information, either using the 'v2_surfaces'
         # or 'v2_impervious_surfaces' table.
         progress_callback(0.95, "Processing 0D domain...")
-        if grid_settings.use_0d_inflow == InflowType.SURFACE.value:
-            surfaces: Surfaces = db.get_surfaces()
-        else:
-            surfaces: ImperviousSurfaces = db.get_impervious_surfaces()
+        surfaces: Surfaces = db.get_surfaces()
         grid.add_0d(surfaces)
 
     grid.finalize()

threedigrid_builder/grid/grid.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, fields
-from typing import Optional, Tuple, Union
+from typing import Optional, Tuple
 
 import numpy as np
 import shapely
@@ -769,9 +769,9 @@ def set_breach_ids(self, breach_points: PotentialBreachPoints):
         breach_points.assign_to_connection_nodes(self.nodes, self.lines)
         breach_points.match_breach_ids_with_calculation_types(self.nodes)
 
-    def add_0d(self, surfaces: Union[zero_d.Surfaces, zero_d.ImperviousSurfaces]):
+    def add_0d(self, surfaces: zero_d.Surfaces):
         """
-        Zero dimension admin derived from 'v2_surfaces' and 'v2_impervious_surfaces'.
+        Zero dimension admin derived from 'surfaces'.
         """
         self.surfaces = surfaces.as_grid_surfaces()
         self.surface_maps = surfaces.as_surface_maps(self.nodes, self.nodes_embedded)

threedigrid_builder/grid/zero_d.py

@@ -6,9 +6,9 @@
 
 from threedigrid_builder.base import Array, surfaces
 from threedigrid_builder.base.nodes import Nodes
-from threedigrid_builder.constants import ContentType, SurfaceClass
+from threedigrid_builder.constants import ContentType
 
-__all__ = ["Surfaces", "ImperviousSurfaces"]
+__all__ = ["Surfaces"]
 
 
 SURFACE_TYPE_PROPERTIES = {
@@ -207,10 +207,8 @@ class BaseSurface:
     surface_id: int
     code: str
     display_name: str
-    nr_of_inhabitants: float
     area: float
-    dry_weather_flow: float
-    the_geom: shapely.Geometry
+    geom: shapely.Geometry
 
     connection_node_id: int
     connection_node_the_geom: shapely.Geometry
@@ -219,31 +217,19 @@ class BaseSurface:
 
 class Surface(BaseSurface):
     """
-    Datamodel based on "v2_surface" table
+    Datamodel based on "surface" table
     """
 
     id: int
-    function: str
     outflow_delay: float
     surface_layer_thickness: float
-    infiltration: float
+    infiltration: bool
     max_infiltration_capacity: float
     min_infiltration_capacity: float
     infiltration_decay_constant: float
     infiltration_recovery_constant: float
 
 
-class ImperviousSurface(BaseSurface):
-    """
-    Datamodel based on "v2_impervious_surface" table
-    """
-
-    id: int
-    surface_inclination: str
-    surface_class: SurfaceClass
-    surface_sub_class: str
-
-
 def fill_missing_centroids(
     centroids: np.ndarray, surface_ids: np.ndarray, connection_node_the_geom: np.ndarray
 ):
@@ -281,7 +267,7 @@ def as_grid_surfaces(
         if extra_fields is None:
             extra_fields = {}
 
-        centroids = shapely.centroid(self.the_geom)
+        centroids = shapely.centroid(self.geom)
         no_centroid_mask = shapely.is_missing(centroids)
 
         if np.any(no_centroid_mask):
@@ -298,8 +284,6 @@ def as_grid_surfaces(
             area=self.area[self.unique_surfaces_mask],
             centroid_x=centroid_coords[:, 0][self.unique_surfaces_mask],
             centroid_y=centroid_coords[:, 1][self.unique_surfaces_mask],
-            dry_weather_flow=self.dry_weather_flow[self.unique_surfaces_mask],
-            nr_of_inhabitants=self.nr_of_inhabitants[self.unique_surfaces_mask],
             **extra_fields
         )
 
@@ -372,37 +356,10 @@ def as_grid_surfaces(self) -> surfaces.Surfaces:
             outflow_delay=self.outflow_delay[unique_surfaces_mask] / 60.0,
             storage_limit=self.surface_layer_thickness[unique_surfaces_mask] / 1000.0,
             infiltration_flag=self.infiltration[unique_surfaces_mask].astype("bool"),
-            function=self.function[unique_surfaces_mask],
             fb=fb,
             fe=fe,
             ka=ka,
             kh=kh,
         )
 
         return super().as_grid_surfaces(extra_fields)
-
-
-class ImperviousSurfaces(Array[ImperviousSurface], BaseSurfaces):
-    def as_grid_surfaces(self) -> surfaces.Surfaces:
-        params = SurfaceParams.from_surface_class_and_inclination(
-            self.surface_class[self.unique_surfaces_mask],
-            self.surface_inclination[self.unique_surfaces_mask],
-        )
-        surface_sub_class = self.surface_sub_class[self.unique_surfaces_mask]
-        surface_sub_class[shapely.is_missing(surface_sub_class)] = b""
-
-        extra_fields = dict(
-            surface_inclination=self.surface_inclination[self.unique_surfaces_mask],
-            surface_class=self.surface_class[self.unique_surfaces_mask],
-            surface_sub_class=surface_sub_class,
-            function=np.full(len(self.unique_surfaces_mask), b""),
-            outflow_delay=params.outflow_delay,
-            storage_limit=params.surface_storage,
-            infiltration_flag=params.infiltration,
-            fb=params.fb,
-            fe=params.fe,
-            ka=params.ka,
-            kh=params.kh,
-        )
-
-        return super().as_grid_surfaces(extra_fields)

threedigrid_builder/interface/db.py

@@ -29,7 +29,6 @@
     DemAverageAreas,
     ExchangeLines,
     GridRefinements,
-    ImperviousSurfaces,
     Obstacles,
     Orifices,
     Pipes,
@@ -44,7 +43,7 @@
 # hardcoded source projection
 SOURCE_EPSG = 4326
 
-MIN_SQLITE_VERSION = 222
+MIN_SQLITE_VERSION = 223
 
 DAY_IN_SECONDS = 24.0 * 3600.0
 
@@ -299,13 +298,10 @@ def get_surfaces(self) -> Surfaces:
             arr = (
                 session.query(
                     models.Surface.id.label("surface_id"),
-                    models.Surface.function,
                     models.Surface.code,
                     models.Surface.display_name,
-                    models.Surface.nr_of_inhabitants,
                     models.Surface.area,
-                    models.Surface.dry_weather_flow,
-                    models.Surface.the_geom,
+                    models.Surface.geom,
                     models.SurfaceParameter.outflow_delay,
                     models.SurfaceParameter.surface_layer_thickness,
                     models.SurfaceParameter.infiltration,
@@ -331,7 +327,7 @@ def get_surfaces(self) -> Surfaces:
             )
 
         # reproject
-        arr["the_geom"] = self.reproject(arr["the_geom"])
+        arr["geom"] = self.reproject(arr["geom"])
         arr["connection_node_the_geom"] = self.reproject(
             arr["connection_node_the_geom"]
         )
@@ -341,50 +337,6 @@ def get_surfaces(self) -> Surfaces:
             **{name: arr[name] for name in arr.dtype.names},
         )
 
-    def get_impervious_surfaces(self) -> ImperviousSurfaces:
-        with self.get_session() as session:
-            arr = (
-                session.query(
-                    models.ImperviousSurface.id.label("surface_id"),
-                    models.ImperviousSurface.code,
-                    models.ImperviousSurface.display_name,
-                    models.ImperviousSurface.surface_inclination,
-                    models.ImperviousSurface.surface_class,
-                    models.ImperviousSurface.surface_sub_class,
-                    models.ImperviousSurface.nr_of_inhabitants,
-                    models.ImperviousSurface.area,
-                    models.ImperviousSurface.dry_weather_flow,
-                    models.ImperviousSurface.the_geom,
-                    models.ConnectionNode.id.label("connection_node_id"),
-                    models.ConnectionNode.the_geom.label("connection_node_the_geom"),
-                    models.ImperviousSurfaceMap.percentage,
-                )
-                .select_from(models.ImperviousSurface)
-                .join(models.ImperviousSurfaceMap)
-                .join(
-                    models.ConnectionNode,
-                    models.ImperviousSurfaceMap.connection_node_id
-                    == models.ConnectionNode.id,
-                )
-                .order_by(models.ImperviousSurface.id)
-                .as_structarray()
-            )
-
-        # convert enums to values
-        arr["surface_class"] = [x.value for x in arr["surface_class"]]
-        arr["surface_inclination"] = [x.value for x in arr["surface_inclination"]]
-
-        # reproject
-        arr["the_geom"] = self.reproject(arr["the_geom"])
-        arr["connection_node_the_geom"] = self.reproject(
-            arr["connection_node_the_geom"]
-        )
-
-        return ImperviousSurfaces(
-            id=np.arange(0, len(arr["surface_id"] + 1), dtype=int),
-            **{name: arr[name] for name in arr.dtype.names},
-        )
-
     def get_boundary_conditions_1d(self) -> BoundaryConditions1D:
         """Return BoundaryConditions1D"""
         with self.get_session() as session:

threedigrid_builder/tests/conftest.py

@@ -33,7 +33,7 @@
 @pytest.fixture(scope="session")
 def db(tmp_path_factory):
     """Yields a threedigrid_builder.interface.db.SQLite object with access
-    to the test v2_bergermeer.sqlite."""
+    to the test v2_bergermeer.gpkg."""
     fn = tmp_path_factory.mktemp("data") / "v2_bergermeer.gpkg"
     sqlite_path = data_path / "v2_bergermeer.gpkg"
     shutil.copyfile(sqlite_path, fn)

threedigrid_builder/tests/test_db.py

@@ -37,7 +37,7 @@ def test_init(tmp_path):
     with mock.patch(
         "threedigrid_builder.interface.db.ThreediDatabase"
     ) as db, mock.patch.object(SQLite, "get_version") as get_version:
-        get_version.return_value = 222
+        get_version.return_value = 223
         sqlite = SQLite(path)
 
     db.assert_called_with(path)
@@ -65,7 +65,7 @@ def test_init_bad_version(tmp_path):
 
 
 def test_get_version(db):
-    assert db.get_version() == 222
+    assert db.get_version() == 223
 
 
 def test_get_boundary_conditions_1d(db):
@@ -86,6 +86,7 @@ def test_get_boundary_conditions_2d(db):
     assert len(boundary_conditions_2d) == 0
 
 
+# TODO: fix this, no clue why this fails
 def test_get_channels(db):
     channels = db.get_channels()
     assert isinstance(channels, Channels)
@@ -391,12 +392,6 @@ def test_get_surfaces(db):
     assert len(surfaces) == 0
 
 
-def test_get_impervious_surfaces(db):
-    impervious_surfaces = db.get_impervious_surfaces()
-    # No impervious_surfaces in test dataset
-    assert len(impervious_surfaces) == 0
-
-
 def test_get_windshieldings(db):
     windshieldings = db.get_windshieldings()
     # No windshielding in test dataset