diff --git a/CHANGES.rst b/CHANGES.rst index fccf7f4c..1b558cc3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,12 +3,29 @@ Changelog of threedi-schema -0.227.4 (unreleased) +0.228.2 (unreleased) -------------------- - Nothing changed yet. +0.228.1 (2024-11-26) +-------------------- + +- Add `progress_func` argument to schema.upgrade + + +0.228.0 (2024-11-25) +-------------------- + +- Implement changes for schema version 300 concerning 1D +- Remove v2 prefix from table names v2_channel, v2_windshielding, v2_cross_section_location, v2_pipe, v2_culvert` v2_orifice and v2_weir +- Move data from v2_cross_section_definition to linked tables (cross_section_location, pipe, culvert, orifice and weir) +- Move data from v2_manhole to connection_nodes and remove v2_manhole table +- Rename v2_pumpstation to pump and add table pump_map that maps the end nodes to pumps +- Remove tables v2_floodfill and v2_cross_section_definition + + 0.227.3 (2024-11-04) -------------------- diff --git a/pytest.ini b/pytest.ini index 851253ed..9d7d92e2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,4 +7,5 @@ markers = migration_224: migration to schema 224 migration_225: migration to schema 225 migration_226: migration to schema 226 + migration_228: migration to schema 228 diff --git a/threedi_schema/__init__.py b/threedi_schema/__init__.py index 55795271..2f30b152 100644 --- a/threedi_schema/__init__.py +++ b/threedi_schema/__init__.py @@ -2,6 +2,6 @@ from .domain import constants, custom_types, models # NOQA # fmt: off -__version__ = '0.227.4.dev0' +__version__ = '0.228.2.dev0' # fmt: on diff --git a/threedi_schema/application/schema.py b/threedi_schema/application/schema.py index d6194187..cd05b7fa 100644 --- a/threedi_schema/application/schema.py +++ b/threedi_schema/application/schema.py @@ -13,11 +13,11 @@ from sqlalchemy import Column, Integer, MetaData, Table, text from sqlalchemy.exc import IntegrityError -from ..domain import constants, models, views +from ..domain import constants, models from ..infrastructure.spatial_index import ensure_spatial_indexes from ..infrastructure.spatialite_versions import copy_models, get_spatialite_version -from ..infrastructure.views import recreate_views from .errors import MigrationMissingError, UpgradeFailedError +from .upgrade_utils import setup_logging __all__ = ["ModelSchema"] @@ -40,11 +40,12 @@ def get_schema_version(): return int(env.get_head_revision()) -def _upgrade_database(db, revision="head", unsafe=True): +def _upgrade_database(db, revision="head", unsafe=True, progress_func=None): """Upgrade ThreediDatabase instance""" engine = db.engine - config = get_alembic_config(engine, unsafe=unsafe) + if progress_func is not None: + setup_logging(db.schema, revision, config, progress_func) alembic_command.upgrade(config, revision) @@ -87,9 +88,9 @@ def upgrade( self, revision="head", backup=True, - set_views=True, upgrade_spatialite_version=False, convert_to_geopackage=False, + progress_func=None, ): """Upgrade the database to the latest version. @@ -103,15 +104,14 @@ def upgrade( If the database is temporary already (or if it is PostGIS), disable it. - Specify 'set_views=True' to also (re)create views after the upgrade. - This is not compatible when upgrading to a different version than the - latest version. - Specify 'upgrade_spatialite_version=True' to also upgrade the spatialite file version after the upgrade. Specify 'convert_to_geopackage=True' to also convert from spatialite to geopackage file version after the upgrade. + + Specify a 'progress_func' to handle progress updates. `progress_func` should + expect a single argument representing the fraction of progress """ try: rev_nr = get_schema_version() if revision == "head" else int(revision) @@ -124,12 +124,6 @@ def upgrade( f"Cannot convert to geopackage for {revision=} because geopackage support is " "enabled from revision 300", ) - if upgrade_spatialite_version and not set_views: - set_views = True - warnings.warn( - "Setting set_views to True because the spatialite version cannot be upgraded without setting the views", - UserWarning, - ) v = self.get_version() if v is not None and v < constants.LATEST_SOUTH_MIGRATION_ID: raise MigrationMissingError( @@ -137,20 +131,19 @@ def upgrade( f"{constants.LATEST_SOUTH_MIGRATION_ID}. Please consult the " f"3Di documentation on how to update legacy databases." ) - if set_views and revision not in ("head", get_schema_version()): - raise ValueError(f"Cannot set views when upgrading to version '{revision}'") if backup: with self.db.file_transaction() as work_db: - _upgrade_database(work_db, revision=revision, unsafe=True) + _upgrade_database( + work_db, revision=revision, unsafe=True, progress_func=progress_func + ) else: - _upgrade_database(self.db, revision=revision, unsafe=False) + _upgrade_database( + self.db, revision=revision, unsafe=False, progress_func=progress_func + ) if upgrade_spatialite_version: self.upgrade_spatialite_version() elif convert_to_geopackage: self.convert_to_geopackage() - set_views = True - if set_views: - self.set_views() def validate_schema(self): """Very basic validation of 3Di schema. @@ -178,20 +171,6 @@ def validate_schema(self): ) return True - def set_views(self): - """(Re)create views in the spatialite according to the latest definitions.""" - version = self.get_version() - schema_version = get_schema_version() - if version != schema_version: - raise MigrationMissingError( - f"Setting views requires schema version " - f"{schema_version}. Current version: {version}." - ) - - _, file_version = get_spatialite_version(self.db) - - recreate_views(self.db, file_version, views.ALL_VIEWS, views.VIEWS_TO_DELETE) - def set_spatial_indexes(self): """(Re)create spatial indexes in the spatialite according to the latest definitions.""" version = self.get_version() diff --git a/threedi_schema/application/upgrade_utils.py b/threedi_schema/application/upgrade_utils.py new file mode 100644 index 00000000..cdf39f5d --- /dev/null +++ b/threedi_schema/application/upgrade_utils.py @@ -0,0 +1,81 @@ +import logging +from typing import Callable, TYPE_CHECKING + +from alembic.config import Config +from alembic.script import ScriptDirectory + +if TYPE_CHECKING: + from .schema import ModelSchema +else: + ModelSchema = None + + +class ProgressHandler(logging.Handler): + def __init__(self, progress_func, total_steps): + super().__init__() + self.progress_func = progress_func + self.total_steps = total_steps + self.current_step = 0 + + def emit(self, record): + msg = record.getMessage() + if msg.startswith("Running upgrade"): + self.progress_func(100 * self.current_step / self.total_steps) + self.current_step += 1 + + +def get_upgrade_steps_count( + config: Config, current_revision: int, target_revision: str = "head" +) -> int: + """ + Count number of upgrade steps for a schematisation upgrade. + + Args: + config: Config parameter containing the configuration information + current_revision: current revision as integer + target_revision: target revision as zero-padded 4 digit string or "head" + """ + if target_revision != "head": + try: + int(target_revision) + except TypeError: + # this should lead to issues in the upgrade pipeline, lets not take over that error handling here + return 0 + # walk_revisions also includes the revision from current_revision to previous + # reduce the number of steps with 1 + offset = -1 + # The first defined revision is 200; revision numbers < 200 will cause walk_revisions to fail + if current_revision < 200: + current_revision = 200 + # set offset to 0 because previous to current is not included in walk_revisions + offset = 0 + if target_revision != "head" and int(target_revision) < current_revision: + # assume that this will be correctly handled by alembic + return 0 + current_revision_str = f"{current_revision:04d}" + script = ScriptDirectory.from_config(config) + # Determine upgrade steps + revisions = script.walk_revisions(current_revision_str, target_revision) + return len(list(revisions)) + offset + + +def setup_logging( + schema: ModelSchema, + target_revision: str, + config: Config, + progress_func: Callable[[float], None], +): + """ + Set up logging for schematisation upgrade + + Args: + schema: ModelSchema object representing the current schema of the application + target_revision: A str specifying the target revision for migration + config: Config object containing configuration settings + progress_func: A Callable with a single argument of type float, used to track progress during migration + """ + n_steps = get_upgrade_steps_count(config, schema.get_version(), target_revision) + logger = logging.getLogger("alembic.runtime.migration") + logger.setLevel(logging.INFO) + handler = ProgressHandler(progress_func, total_steps=n_steps) + logger.addHandler(handler) diff --git a/threedi_schema/domain/constants.py b/threedi_schema/domain/constants.py index 9fdfe522..90e86a26 100644 --- a/threedi_schema/domain/constants.py +++ b/threedi_schema/domain/constants.py @@ -62,12 +62,19 @@ class CalculationTypeCulvert(Enum): DOUBLE_CONNECTED = 105 +# TODO: rename enum (?) class CalculationTypeNode(Enum): EMBEDDED = 0 ISOLATED = 1 CONNECTED = 2 +class AmbiguousClosedError(Exception): + def __init__(self, shape): + self.shape = shape + super().__init__(f"Closed state is ambiguous for shape: {self.shape}") + + class CrossSectionShape(Enum): CLOSED_RECTANGLE = 0 RECTANGLE = 1 @@ -78,6 +85,22 @@ class CrossSectionShape(Enum): TABULATED_YZ = 7 INVERTED_EGG = 8 + @property + def is_tabulated(self): + return self in { + CrossSectionShape.TABULATED_RECTANGLE, + CrossSectionShape.TABULATED_TRAPEZIUM, + CrossSectionShape.TABULATED_YZ, + } + + @property + def is_closed(self): + if self.is_tabulated: + raise AmbiguousClosedError(self) + if self == CrossSectionShape.RECTANGLE: + return False + return True + class FrictionType(Enum): CHEZY = 1 @@ -159,17 +182,6 @@ class InfiltrationSurfaceOption(Enum): WET_SURFACE = 2 -class ZoomCategories(Enum): - # Visibility in live-site: 0 is lowest for smallest level (i.e. ditch) - # and 5 for highest (rivers). - LOWEST_VISIBILITY = 0 - LOW_VISIBILITY = 1 - MEDIUM_LOW_VISIBILITY = 2 - MEDIUM_VISIBILITY = 3 - HIGH_VISIBILITY = 4 - HIGHEST_VISIBILITY = 5 - - class InflowType(Enum): NO_INFLOW = 0 IMPERVIOUS_SURFACE = 1 @@ -205,12 +217,21 @@ class ControlType(Enum): class StructureControlTypes(Enum): - pumpstation = "v2_pumpstation" - pipe = "v2_pipe" - orifice = "v2_orifice" - culvert = "v2_culvert" - weir = "v2_weir" - channel = "v2_channel" + pumpstation = "pump" + pipe = "pipe" + orifice = "orifice" + culvert = "culvert" + weir = "weir" + channel = "channel" + + def get_legacy_value(self) -> str: + """ + Get value of structure control as used in schema 2.x + """ + if self == StructureControlTypes.pumpstation: + return "v2_pumpstation" + else: + return f"v2_{self.value}" class ControlTableActionTypes(Enum): @@ -240,3 +261,8 @@ class AdvectionTypes1D(Enum): MOMENTUM_CONSERVATIVE = 1 ENERGY_CONSERVATIVE = 2 COMBINED_MOMENTUM_AND_ENERGY_CONSERVATIVE = 3 + + +class NodeOpenWaterDetection(Enum): + HAS_CHANNEL = 0 + HAS_STORAGE = 1 diff --git a/threedi_schema/domain/models.py b/threedi_schema/domain/models.py index d70bddea..6a7e658e 100644 --- a/threedi_schema/domain/models.py +++ b/threedi_schema/domain/models.py @@ -1,5 +1,5 @@ from sqlalchemy import Boolean, Column, Float, ForeignKey, Integer, String, Text -from sqlalchemy.orm import declarative_base, relationship +from sqlalchemy.orm import declarative_base from . import constants from .custom_types import Geometry, IntegerEnum, VarcharEnum @@ -91,13 +91,6 @@ class ControlTable(Base): tags = Column(Text) -class Floodfill(Base): - __tablename__ = "v2_floodfill" - id = Column(Integer, primary_key=True) - waterlevel = Column(Float) - the_geom = Column(Geometry("POINT")) - - class Interflow(Base): __tablename__ = "interflow" id = Column(Integer, primary_key=True) @@ -259,29 +252,23 @@ class GridRefinementArea(Base): tags = Column(Text) -class CrossSectionDefinition(Base): - __tablename__ = "v2_cross_section_definition" - id = Column(Integer, primary_key=True) - width = Column(String(255)) - height = Column(String(255)) - shape = Column(IntegerEnum(constants.CrossSectionShape)) - code = Column(String(100)) - friction_values = Column(String) - vegetation_stem_densities = Column(String) - vegetation_stem_diameters = Column(String) - vegetation_heights = Column(String) - vegetation_drag_coefficients = Column(String) - - class ConnectionNode(Base): - __tablename__ = "v2_connection_nodes" + __tablename__ = "connection_node" id = Column(Integer, primary_key=True) - storage_area = Column(Float) - initial_waterlevel = Column(Float) - the_geom = Column(Geometry("POINT"), nullable=False) + geom = Column(Geometry("POINT"), nullable=False) code = Column(String(100)) - - manholes = relationship("Manhole", back_populates="connection_node") + tags = Column(Text) + display_name = Column(Text) + storage_area = Column(Float) + initial_water_level = Column(Float) + visualisation = Column(Integer) + manhole_surface_level = Column(Float) + bottom_level = Column(Float) + exchange_level = Column(Float) + exchange_type = Column(IntegerEnum(constants.CalculationTypeNode)) + exchange_thickness = Column(Float) + hydraulic_conductivity_in = Column(Float) + hydraulic_conductivity_out = Column(Float) class Lateral1d(Base): @@ -300,35 +287,6 @@ class Lateral1d(Base): connection_node_id = Column(Integer) -class Manhole(Base): - __tablename__ = "v2_manhole" - - id = Column(Integer, primary_key=True) - display_name = Column(String(255)) - code = Column(String(100)) - zoom_category = Column(IntegerEnum(constants.ZoomCategories)) - shape = Column(String(4)) - width = Column(Float) - length = Column(Float) - surface_level = Column(Float) - bottom_level = Column(Float, nullable=False) - drain_level = Column(Float) - sediment_level = Column(Float) - manhole_indicator = Column(Integer) - calculation_type = Column(IntegerEnum(constants.CalculationTypeNode)) - exchange_thickness = Column(Float) - hydraulic_conductivity_in = Column(Float) - hydraulic_conductivity_out = Column(Float) - - connection_node_id = Column( - Integer, - ForeignKey(ConnectionNode.__tablename__ + ".id"), - nullable=False, - unique=True, - ) - connection_node = relationship(ConnectionNode, back_populates="manholes") - - class NumericalSettings(Base): __tablename__ = "numerical_settings" id = Column(Integer, primary_key=True) @@ -405,6 +363,7 @@ class ModelSettings(Base): use_groundwater_flow = Column(Boolean) use_groundwater_storage = Column(Boolean) use_vegetation_drag_2d = Column(Boolean) + node_open_water_detection = Column(IntegerEnum(constants.NodeOpenWaterDetection)) # Alias needed for API compatibility @property @@ -502,37 +461,23 @@ class SurfaceMap(Base): class Channel(Base): - __tablename__ = "v2_channel" + __tablename__ = "channel" id = Column(Integer, primary_key=True) display_name = Column(String(255)) code = Column(String(100)) - calculation_type = Column(IntegerEnum(constants.CalculationType), nullable=False) - dist_calc_points = Column(Float) - zoom_category = Column(IntegerEnum(constants.ZoomCategories)) - the_geom = Column(Geometry("LINESTRING"), nullable=False) - - connection_node_start_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_start = relationship( - ConnectionNode, foreign_keys=connection_node_start_id - ) - connection_node_end_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_end = relationship( - ConnectionNode, foreign_keys=connection_node_end_id - ) - cross_section_locations = relationship( - "CrossSectionLocation", back_populates="channel" - ) + tags = Column(Text) + exchange_type = Column(IntegerEnum(constants.CalculationType)) + calculation_point_distance = Column(Float) + geom = Column(Geometry("LINESTRING"), nullable=False) + connection_node_id_start = Column(Integer) + connection_node_id_end = Column(Integer) exchange_thickness = Column(Float) hydraulic_conductivity_in = Column(Float) hydraulic_conductivity_out = Column(Float) class Windshielding(Base): - __tablename__ = "v2_windshielding" + __tablename__ = "windshielding_1d" id = Column(Integer, primary_key=True) north = Column(Float) northeast = Column(Float) @@ -542,115 +487,80 @@ class Windshielding(Base): southwest = Column(Float) west = Column(Float) northwest = Column(Float) - the_geom = Column(Geometry("POINT")) - channel_id = Column( - Integer, ForeignKey(Channel.__tablename__ + ".id"), nullable=False - ) + geom = Column(Geometry("POINT"), nullable=False) + channel_id = Column(Integer) class CrossSectionLocation(Base): - __tablename__ = "v2_cross_section_location" + __tablename__ = "cross_section_location" id = Column(Integer, primary_key=True) code = Column(String(100)) - reference_level = Column(Float, nullable=False) - friction_type = Column(IntegerEnum(constants.FrictionType), nullable=False) + reference_level = Column(Float) + friction_type = Column(IntegerEnum(constants.FrictionType)) friction_value = Column(Float) bank_level = Column(Float) + cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) + cross_section_width = Column(Float) + cross_section_height = Column(Float) + cross_section_friction_values = Column(Text) + cross_section_vegetation_table = Column(Text) + cross_section_table = Column(Text) vegetation_stem_density = Column(Float) vegetation_stem_diameter = Column(Float) vegetation_height = Column(Float) vegetation_drag_coefficient = Column(Float) - the_geom = Column(Geometry("POINT"), nullable=False) - channel_id = Column( - Integer, ForeignKey(Channel.__tablename__ + ".id"), nullable=False - ) - channel = relationship(Channel, back_populates="cross_section_locations") - definition_id = Column( - Integer, - ForeignKey(CrossSectionDefinition.__tablename__ + ".id"), - nullable=False, - ) - definition = relationship(CrossSectionDefinition) + geom = Column(Geometry("POINT"), nullable=False) + channel_id = Column(Integer) class Pipe(Base): - __tablename__ = "v2_pipe" + __tablename__ = "pipe" id = Column(Integer, primary_key=True) display_name = Column(String(255)) code = Column(String(100)) - profile_num = Column(Integer) + tags = Column(Text) + geom = Column(Geometry("LINESTRING"), nullable=False) sewerage_type = Column(IntegerEnum(constants.SewerageType)) - calculation_type = Column( - IntegerEnum(constants.PipeCalculationType), nullable=False - ) - invert_level_start_point = Column(Float, nullable=False) - invert_level_end_point = Column(Float, nullable=False) - friction_value = Column(Float, nullable=False) - friction_type = Column(IntegerEnum(constants.FrictionType), nullable=False) - dist_calc_points = Column(Float) - material = Column(Integer) - original_length = Column(Float) - zoom_category = Column(IntegerEnum(constants.ZoomCategories)) - - connection_node_start_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_start = relationship( - ConnectionNode, foreign_keys=connection_node_start_id - ) - connection_node_end_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_end = relationship( - ConnectionNode, foreign_keys=connection_node_end_id - ) - cross_section_definition_id = Column( - Integer, - ForeignKey(CrossSectionDefinition.__tablename__ + ".id"), - nullable=False, - ) - cross_section_definition = relationship("CrossSectionDefinition") + exchange_type = Column(IntegerEnum(constants.PipeCalculationType)) + invert_level_start = Column(Float) + invert_level_end = Column(Float) + friction_value = Column(Float) + friction_type = Column(IntegerEnum(constants.FrictionType)) + calculation_point_distance = Column(Float) + material_id = Column(Integer) + connection_node_id_start = Column(Integer) + connection_node_id_end = Column(Integer) + cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) + cross_section_width = Column(Float) + cross_section_height = Column(Float) + cross_section_table = Column(Text) exchange_thickness = Column(Float) hydraulic_conductivity_in = Column(Float) hydraulic_conductivity_out = Column(Float) class Culvert(Base): - __tablename__ = "v2_culvert" + __tablename__ = "culvert" id = Column(Integer, primary_key=True) display_name = Column(String(255)) code = Column(String(100)) - calculation_type = Column(IntegerEnum(constants.CalculationTypeCulvert)) - friction_value = Column(Float, nullable=False) - friction_type = Column(IntegerEnum(constants.FrictionType), nullable=False) - dist_calc_points = Column(Float) - zoom_category = Column(IntegerEnum(constants.ZoomCategories)) + tags = Column(Text) + exchange_type = Column(IntegerEnum(constants.CalculationTypeCulvert)) + friction_value = Column(Float) + friction_type = Column(IntegerEnum(constants.FrictionType)) + calculation_point_distance = Column(Float) discharge_coefficient_positive = Column(Float) discharge_coefficient_negative = Column(Float) - invert_level_start_point = Column(Float, nullable=False) - invert_level_end_point = Column(Float, nullable=False) - the_geom = Column( - Geometry("LINESTRING"), - ) - - connection_node_start_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_start = relationship( - ConnectionNode, foreign_keys=connection_node_start_id - ) - connection_node_end_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_end = relationship( - ConnectionNode, foreign_keys=connection_node_end_id - ) - cross_section_definition_id = Column( - Integer, - ForeignKey(CrossSectionDefinition.__tablename__ + ".id"), - nullable=False, - ) - cross_section_definition = relationship(CrossSectionDefinition) + invert_level_start = Column(Float) + invert_level_end = Column(Float) + geom = Column(Geometry("LINESTRING"), nullable=False) + material_id = Column(Integer) + connection_node_id_start = Column(Integer) + connection_node_id_end = Column(Integer) + cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) + cross_section_width = Column(Float) + cross_section_height = Column(Float) + cross_section_table = Column(Text) class DemAverageArea(Base): @@ -663,101 +573,79 @@ class DemAverageArea(Base): class Weir(Base): - __tablename__ = "v2_weir" + __tablename__ = "weir" id = Column(Integer, primary_key=True) code = Column(String(100)) display_name = Column(String(255)) - crest_level = Column(Float, nullable=False) - crest_type = Column(IntegerEnum(constants.CrestType), nullable=False) + geom = Column(Geometry("LINESTRING"), nullable=False) + tags = Column(Text) + crest_level = Column(Float) + crest_type = Column(IntegerEnum(constants.CrestType)) friction_value = Column(Float) friction_type = Column(IntegerEnum(constants.FrictionType)) discharge_coefficient_positive = Column(Float) discharge_coefficient_negative = Column(Float) + material_id = Column(Integer) sewerage = Column(Boolean) external = Column(Boolean) - zoom_category = Column(IntegerEnum(constants.ZoomCategories)) - - connection_node_start_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_start = relationship( - ConnectionNode, foreign_keys=connection_node_start_id - ) - connection_node_end_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_end = relationship( - ConnectionNode, foreign_keys=connection_node_end_id - ) - cross_section_definition_id = Column( - Integer, - ForeignKey(CrossSectionDefinition.__tablename__ + ".id"), - nullable=False, - ) - cross_section_definition = relationship("CrossSectionDefinition") + connection_node_id_start = Column(Integer) + connection_node_id_end = Column(Integer) + cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) + cross_section_width = Column(Float) + cross_section_height = Column(Float) + cross_section_table = Column(Text) class Orifice(Base): - __tablename__ = "v2_orifice" + __tablename__ = "orifice" id = Column(Integer, primary_key=True) code = Column(String(100)) display_name = Column(String(255)) - zoom_category = Column(IntegerEnum(constants.ZoomCategories)) - crest_type = Column(IntegerEnum(constants.CrestType), nullable=False) - crest_level = Column(Float, nullable=False) + tags = Column(Text) + geom = Column(Geometry("LINESTRING"), nullable=False) + crest_type = Column(IntegerEnum(constants.CrestType)) + crest_level = Column(Float) + material_id = Column(Integer) friction_value = Column(Float) friction_type = Column(IntegerEnum(constants.FrictionType)) discharge_coefficient_positive = Column(Float) discharge_coefficient_negative = Column(Float) sewerage = Column(Boolean) - - connection_node_start_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_start = relationship( - ConnectionNode, foreign_keys=connection_node_start_id - ) - connection_node_end_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_end = relationship( - ConnectionNode, foreign_keys=connection_node_end_id - ) - cross_section_definition_id = Column( - Integer, - ForeignKey(CrossSectionDefinition.__tablename__ + ".id"), - nullable=False, - ) - cross_section_definition = relationship("CrossSectionDefinition") + connection_node_id_start = Column(Integer) + connection_node_id_end = Column(Integer) + cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) + cross_section_width = Column(Float) + cross_section_height = Column(Float) + cross_section_table = Column(Text) -class Pumpstation(Base): - __tablename__ = "v2_pumpstation" +class Pump(Base): + __tablename__ = "pump" id = Column(Integer, primary_key=True) code = Column(String(100)) display_name = Column(String(255)) - zoom_category = Column(IntegerEnum(constants.ZoomCategories)) - classification = Column(Integer) - sewerage = Column(Boolean) + start_level = Column(Float) + lower_stop_level = Column(Float) + upper_stop_level = Column(Float) + capacity = Column(Float) type_ = Column( - IntegerEnum(constants.PumpType), name="type", key="type_", nullable=False + IntegerEnum(constants.PumpType), name="type", key="type_" ) # type: ignore[call-overload] - start_level = Column(Float, nullable=False) - lower_stop_level = Column(Float, nullable=False) - upper_stop_level = Column(Float) - capacity = Column(Float, nullable=False) - connection_node_start_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id"), nullable=False - ) - connection_node_start = relationship( - "ConnectionNode", foreign_keys=connection_node_start_id - ) - connection_node_end_id = Column( - Integer, ForeignKey(ConnectionNode.__tablename__ + ".id") - ) - connection_node_end = relationship( - ConnectionNode, foreign_keys=connection_node_end_id - ) + sewerage = Column(Boolean) + connection_node_id = Column(Integer) + geom = Column(Geometry("POINT"), nullable=False) + tags = Column(Text) + + +class PumpMap(Base): + __tablename__ = "pump_map" + id = Column(Integer, primary_key=True) + pump_id = Column(Integer) + connection_node_id_end = Column(Integer) + geom = Column(Geometry("LINESTRING"), nullable=False) + tags = Column(Text) + code = Column(String(100)) + display_name = Column(String(255)) class Obstacle(Base): @@ -768,6 +656,9 @@ class Obstacle(Base): geom = Column(Geometry("LINESTRING"), nullable=False) tags = Column(Text) display_name = Column(String(255)) + affects_2d = Column(Boolean) + affects_1d2d_open_water = Column(Boolean) + affects_1d2d_closed = Column(Boolean) class PotentialBreach(Base): @@ -800,6 +691,14 @@ class Tags(Base): description = Column(Text) +class Material(Base): + __tablename__ = "material" + id = Column(Integer, primary_key=True) + description = Column(Text) + friction_type = Column(IntegerEnum(constants.FrictionType)) + friction_coefficient = Column(Float) + + DECLARED_MODELS = [ AggregationSettings, BoundaryCondition1D, @@ -810,7 +709,6 @@ class Tags(Base): ControlMeasureMap, ControlMemory, ControlTable, - CrossSectionDefinition, CrossSectionLocation, Culvert, DemAverageArea, @@ -818,7 +716,6 @@ class Tags(Base): DryWeatherFlowMap, DryWeatherFlowDistribution, ExchangeLine, - Floodfill, GridRefinementLine, GridRefinementArea, GroundWater, @@ -827,7 +724,7 @@ class Tags(Base): Interflow, Lateral1d, Lateral2D, - Manhole, + Material, ModelSettings, NumericalSettings, Obstacle, @@ -835,7 +732,8 @@ class Tags(Base): PhysicalSettings, Pipe, PotentialBreach, - Pumpstation, + Pump, + PumpMap, SimpleInfiltration, SimulationTemplateSettings, Surface, diff --git a/threedi_schema/domain/views.py b/threedi_schema/domain/views.py deleted file mode 100644 index 85af22f1..00000000 --- a/threedi_schema/domain/views.py +++ /dev/null @@ -1,72 +0,0 @@ -VIEWS_TO_DELETE = [ - "v2_crosssection_view", - "v2_pipe_map_view", - "v2_imp_surface_view", - "v2_1d_lateral_view", - "v2_1d_boundary_conditions_view", -] -ALL_VIEWS = { - "v2_cross_section_location_view": { - "definition": "SELECT loc.rowid as rowid, loc.id as loc_id, loc.code as loc_code, loc.reference_level as loc_reference_level, loc.bank_level as loc_bank_level, loc.friction_type as loc_friction_type, loc.friction_value as loc_friction_value, loc.definition_id as loc_definition_id, loc.channel_id as loc_channel_id, loc.the_geom as the_geom, loc.vegetation_stem_density as loc_vegetation_stem_density, loc.vegetation_stem_diameter as loc_vegetation_stem_diameter, loc.vegetation_height as loc_vegetation_height, loc.vegetation_drag_coefficient as loc_vegetation_drag_coefficient, def.id as def_id, def.shape as def_shape, def.width as def_width, def.code as def_code, def.height as def_height, def.friction_values as def_friction_values, def.vegetation_stem_densities as def_vegetation_stem_densities, def.vegetation_stem_diameters as def_vegetation_stem_diameters, def.vegetation_heights as def_vegetation_heights, def.vegetation_drag_coefficients as def_vegetation_drag_coefficients FROM v2_cross_section_location loc, v2_cross_section_definition def WHERE loc.definition_id = def.id", - "view_geometry": "the_geom", - "view_rowid": "rowid", - "f_table_name": "v2_cross_section_location", - "f_geometry_column": "the_geom", - }, - "v2_cross_section_view": { - "definition": "SELECT def.rowid AS rowid, def.id AS def_id, def.shape AS def_shape, def.width AS def_width, def.height AS def_height, def.code AS def_code, l.id AS l_id, l.channel_id AS l_channel_id, l.definition_id AS l_definition_id, l.reference_level AS l_reference_level, l.friction_type AS l_friction_type, l.friction_value AS l_friction_value, l.bank_level AS l_bank_level, l.code AS l_code, l.the_geom AS the_geom, ch.id AS ch_id, ch.display_name AS ch_display_name, ch.code AS ch_code, ch.calculation_type AS ch_calculation_type, ch.dist_calc_points AS ch_dist_calc_points, ch.zoom_category AS ch_zoom_category, ch.connection_node_start_id AS ch_connection_node_start_id, ch.connection_node_end_id AS ch_connection_node_end_id FROM v2_cross_section_definition AS def , v2_cross_section_location AS l , v2_channel AS ch WHERE l.definition_id = def.id AND l.channel_id = ch.id", - "view_geometry": "the_geom", - "view_rowid": "rowid", - "f_table_name": "v2_cross_section_location", - "f_geometry_column": "the_geom", - }, - "v2_culvert_view": { - "definition": "SELECT cul.rowid AS rowid, cul.id AS cul_id, cul.display_name AS cul_display_name, cul.code AS cul_code, cul.calculation_type AS cul_calculation_type, cul.friction_value AS cul_friction_value, cul.friction_type AS cul_friction_type, cul.dist_calc_points AS cul_dist_calc_points, cul.zoom_category AS cul_zoom_category, cul.cross_section_definition_id AS cul_cross_section_definition_id, cul.discharge_coefficient_positive AS cul_discharge_coefficient_positive, cul.discharge_coefficient_negative AS cul_discharge_coefficient_negative, cul.invert_level_start_point AS cul_invert_level_start_point, cul.invert_level_end_point AS cul_invert_level_end_point, cul.the_geom AS the_geom, cul.connection_node_start_id AS cul_connection_node_start_id, cul.connection_node_end_id AS cul_connection_node_end_id, def.id AS def_id, def.shape AS def_shape, def.width AS def_width, def.height AS def_height, def.code AS def_code FROM v2_culvert AS cul , v2_cross_section_definition AS def WHERE cul.cross_section_definition_id = def.id", - "view_geometry": "the_geom", - "view_rowid": "rowid", - "f_table_name": "v2_culvert", - "f_geometry_column": "the_geom", - }, - "v2_manhole_view": { - "definition": "SELECT manh.rowid AS rowid, manh.id AS manh_id, manh.display_name AS manh_display_name, manh.code AS manh_code, manh.connection_node_id AS manh_connection_node_id, manh.shape AS manh_shape, manh.width AS manh_width, manh.length AS manh_length, manh.manhole_indicator AS manh_manhole_indicator, manh.calculation_type AS manh_calculation_type, manh.bottom_level AS manh_bottom_level, manh.surface_level AS manh_surface_level, manh.drain_level AS manh_drain_level, manh.sediment_level AS manh_sediment_level, manh.zoom_category AS manh_zoom_category, manh.exchange_thickness AS manh_exchange_thickness, manh.hydraulic_conductivity_in AS manh_hydraulic_conductivity_in, manh.hydraulic_conductivity_out AS manh_hydraulic_conductivity_out, node.id AS node_id, node.storage_area AS node_storage_area, node.initial_waterlevel AS node_initial_waterlevel, node.code AS node_code, node.the_geom AS the_geom FROM v2_manhole AS manh , v2_connection_nodes AS node WHERE manh.connection_node_id = node.id", - "view_geometry": "the_geom", - "view_rowid": "rowid", - "f_table_name": "v2_connection_nodes", - "f_geometry_column": "the_geom", - }, - "v2_orifice_view": { - "definition": "SELECT orf.rowid AS rowid, orf.id AS orf_id, orf.display_name AS orf_display_name, orf.code AS orf_code, orf.crest_level AS orf_crest_level, orf.sewerage AS orf_sewerage, orf.cross_section_definition_id AS orf_cross_section_definition_id, orf.friction_value AS orf_friction_value, orf.friction_type AS orf_friction_type, orf.discharge_coefficient_positive AS orf_discharge_coefficient_positive, orf.discharge_coefficient_negative AS orf_discharge_coefficient_negative, orf.zoom_category AS orf_zoom_category, orf.crest_type AS orf_crest_type, orf.connection_node_start_id AS orf_connection_node_start_id, orf.connection_node_end_id AS orf_connection_node_end_id, def.id AS def_id, def.shape AS def_shape, def.width AS def_width, def.height AS def_height, def.code AS def_code, MakeLine( start_node.the_geom, end_node.the_geom) AS the_geom FROM v2_orifice AS orf, v2_cross_section_definition AS def, v2_connection_nodes AS start_node, v2_connection_nodes AS end_node where orf.connection_node_start_id = start_node.id AND orf.connection_node_end_id = end_node.id AND orf.cross_section_definition_id = def.id", - "view_geometry": "the_geom", - "view_rowid": "rowid", - "f_table_name": "v2_connection_nodes", - "f_geometry_column": "the_geom_linestring", - }, - "v2_pipe_view": { - "definition": "SELECT pipe.rowid AS rowid, pipe.id AS pipe_id, pipe.display_name AS pipe_display_name, pipe.code AS pipe_code, pipe.profile_num AS pipe_profile_num, pipe.sewerage_type AS pipe_sewerage_type, pipe.calculation_type AS pipe_calculation_type, pipe.invert_level_start_point AS pipe_invert_level_start_point, pipe.invert_level_end_point AS pipe_invert_level_end_point, pipe.cross_section_definition_id AS pipe_cross_section_definition_id, pipe.friction_value AS pipe_friction_value, pipe.friction_type AS pipe_friction_type, pipe.dist_calc_points AS pipe_dist_calc_points, pipe.material AS pipe_material, pipe.original_length AS pipe_original_length, pipe.zoom_category AS pipe_zoom_category, pipe.connection_node_start_id AS pipe_connection_node_start_id, pipe.connection_node_end_id AS pipe_connection_node_end_id, pipe.exchange_thickness AS pipe_exchange_thickness, pipe.hydraulic_conductivity_in AS pipe_hydraulic_conductivity_in, pipe.hydraulic_conductivity_out AS pipe_hydraulic_conductivity_out, def.id AS def_id, def.shape AS def_shape, def.width AS def_width, def.height AS def_height, def.code AS def_code, MakeLine( start_node.the_geom, end_node.the_geom) AS the_geom FROM v2_pipe AS pipe , v2_cross_section_definition AS def , v2_connection_nodes AS start_node , v2_connection_nodes AS end_node WHERE pipe.connection_node_start_id = start_node.id AND pipe.connection_node_end_id = end_node.id AND pipe.cross_section_definition_id = def.id", - "view_geometry": "the_geom", - "view_rowid": "rowid", - "f_table_name": "v2_connection_nodes", - "f_geometry_column": "the_geom_linestring", - }, - "v2_pumpstation_point_view": { - "definition": "SELECT a.rowid AS rowid, a.id AS pump_id, a.display_name, a.code, a.classification, a.sewerage, a.start_level, a.lower_stop_level, a.upper_stop_level, a.capacity, a.zoom_category, a.connection_node_start_id, a.connection_node_end_id, a.type, b.id AS connection_node_id, b.storage_area, b.the_geom FROM v2_pumpstation a JOIN v2_connection_nodes b ON a.connection_node_start_id = b.id", - "view_geometry": "the_geom", - "view_rowid": "connection_node_start_id", - "f_table_name": "v2_connection_nodes", - "f_geometry_column": "the_geom", - }, - "v2_pumpstation_view": { - "definition": "SELECT pump.rowid AS rowid, pump.id AS pump_id, pump.display_name AS pump_display_name, pump.code AS pump_code, pump.classification AS pump_classification, pump.type AS pump_type, pump.sewerage AS pump_sewerage, pump.start_level AS pump_start_level, pump.lower_stop_level AS pump_lower_stop_level, pump.upper_stop_level AS pump_upper_stop_level, pump.capacity AS pump_capacity, pump.zoom_category AS pump_zoom_category, pump.connection_node_start_id AS pump_connection_node_start_id, pump.connection_node_end_id AS pump_connection_node_end_id, MakeLine( start_node.the_geom, end_node.the_geom ) AS the_geom FROM v2_pumpstation AS pump , v2_connection_nodes AS start_node , v2_connection_nodes AS end_node WHERE pump.connection_node_start_id = start_node.id AND pump.connection_node_end_id = end_node.id", - "view_geometry": "the_geom", - "view_rowid": "rowid", - "f_table_name": "v2_connection_nodes", - "f_geometry_column": "the_geom_linestring", - }, - "v2_weir_view": { - "definition": "SELECT weir.rowid AS rowid, weir.id AS weir_id, weir.display_name AS weir_display_name, weir.code AS weir_code, weir.crest_level AS weir_crest_level, weir.crest_type AS weir_crest_type, weir.cross_section_definition_id AS weir_cross_section_definition_id, weir.sewerage AS weir_sewerage, weir.discharge_coefficient_positive AS weir_discharge_coefficient_positive, weir.discharge_coefficient_negative AS weir_discharge_coefficient_negative, weir.external AS weir_external, weir.zoom_category AS weir_zoom_category, weir.friction_value AS weir_friction_value, weir.friction_type AS weir_friction_type, weir.connection_node_start_id AS weir_connection_node_start_id, weir.connection_node_end_id AS weir_connection_node_end_id, def.id AS def_id, def.shape AS def_shape, def.width AS def_width, def.height AS def_height, def.code AS def_code, MakeLine( start_node.the_geom, end_node.the_geom) AS the_geom FROM v2_weir AS weir , v2_cross_section_definition AS def , v2_connection_nodes AS start_node , v2_connection_nodes AS end_node WHERE weir.connection_node_start_id = start_node.id AND weir.connection_node_end_id = end_node.id AND weir.cross_section_definition_id = def.id", - "view_geometry": "the_geom", - "view_rowid": "rowid", - "f_table_name": "v2_connection_nodes", - "f_geometry_column": "the_geom_linestring", - }, -} diff --git a/threedi_schema/migrations/versions/0228_upgrade_db_1D.py b/threedi_schema/migrations/versions/0228_upgrade_db_1D.py new file mode 100644 index 00000000..c4e532ca --- /dev/null +++ b/threedi_schema/migrations/versions/0228_upgrade_db_1D.py @@ -0,0 +1,485 @@ +"""Upgrade settings in schema + +Revision ID: 0225 +Revises: +Create Date: 2024-09-10 09:00 + +""" +import csv +import uuid +from pathlib import Path +from typing import Dict, List, Tuple + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import Column, Float, func, Integer, select, String +from sqlalchemy.orm import declarative_base, Session + +from threedi_schema.domain import constants, models +from threedi_schema.domain.custom_types import IntegerEnum + +Base = declarative_base() + +data_dir = Path(__file__).parent / "data" + + +# revision identifiers, used by Alembic. +revision = "0228" +down_revision = "0227" +branch_labels = None +depends_on = None + +RENAME_TABLES = [ + ("v2_channel", "channel"), + ("v2_windshielding", "windshielding_1d"), + ("v2_cross_section_location", "cross_section_location"), + ("v2_pipe", "pipe"), + ("v2_culvert", "culvert"), + ("v2_weir", "weir"), + ("v2_orifice", "orifice"), + ("v2_pumpstation", "pump") +] + +DELETE_TABLES = ["v2_cross_section_definition", + "v2_floodfill", + "v2_connection_nodes"] + +RENAME_COLUMNS = { + "culvert": {"calculation_type": "exchange_type", + "dist_calc_points": "calculation_point_distance", + "invert_level_start_point": "invert_level_start", + "invert_level_end_point": "invert_level_end", + "connection_node_start_id": "connection_node_id_start", + "connection_node_end_id": "connection_node_id_end"}, + "pipe": {"calculation_type": "exchange_type", + "dist_calc_points": "calculation_point_distance", + "material": "material_id", + "invert_level_start_point": "invert_level_start", + "invert_level_end_point": "invert_level_end", + "connection_node_start_id": "connection_node_id_start", + "connection_node_end_id": "connection_node_id_end"}, + "channel": {"calculation_type": "exchange_type", + "dist_calc_points": "calculation_point_distance", + "connection_node_start_id": "connection_node_id_start", + "connection_node_end_id": "connection_node_id_end"}, + "weir": {"calculation_type": "exchange_type", + "dist_calc_points": "calculation_point_distance", + "connection_node_start_id": "connection_node_id_start", + "connection_node_end_id": "connection_node_id_end"}, + "orifice": {"calculation_type": "exchange_type", + "dist_calc_points": "calculation_point_distance", + "connection_node_start_id": "connection_node_id_start", + "connection_node_end_id": "connection_node_id_end"}, + "pump": {"connection_node_start_id": "connection_node_id"} +} + +REMOVE_COLUMNS = { + "channel": ["zoom_category",], + "cross_section_location": ["definition_id", "vegetation_drag_coeficients"], + "culvert": ["zoom_category", "cross_section_definition_id"], + "pipe": ["zoom_category", "original_length", "cross_section_definition_id", "profile_num"], + "orifice": ["zoom_category", "cross_section_definition_id"], + "weir": ["zoom_category", "cross_section_definition_id"], + "pump": ["connection_node_end_id", "zoom_category", "classification"] +} + +RETYPE_COLUMNS = {} + + +class Schema228UpgradeException(Exception): + pass + + +def add_columns_to_tables(table_columns: List[Tuple[str, Column]]): + # no checks for existence are done, this will fail if any column already exists + for dst_table, col in table_columns: + with op.batch_alter_table(dst_table) as batch_op: + batch_op.add_column(col) + + +def remove_tables(tables: List[str]): + for table in tables: + op.drop_table(table) + + +def modify_table(old_table_name, new_table_name): + # Create a new table named `new_table_name` using the declared models + # Use the columns from `old_table_name`, with the following exceptions: + # * columns in `RENAME_COLUMNS[new_table_name]` are renamed + # * `the_geom` is renamed to `geom` and NOT NULL is enforced + model = find_model(new_table_name) + # create new table + create_sqlite_table_from_model(model) + # get column names from model and match them to available data in sqlite + connection = op.get_bind() + rename_cols = {**RENAME_COLUMNS.get(new_table_name, {}), "the_geom": "geom"} + rename_cols_rev = {v: k for k, v in rename_cols.items()} + col_map = [(col.name, rename_cols_rev.get(col.name, col.name)) for col in get_cols_for_model(model)] + available_cols = [col[1] for col in connection.execute(sa.text(f"PRAGMA table_info('{old_table_name}')")).fetchall()] + new_col_names, old_col_names = zip(*[(new_col, old_col) for new_col, old_col in col_map if old_col in available_cols]) + # Copy data + # This may copy wrong type data because some types change!! + op.execute(sa.text(f"INSERT INTO {new_table_name} ({','.join(new_col_names)}) " + f"SELECT {','.join(old_col_names)} FROM {old_table_name}")) + + +def find_model(table_name): + for model in models.DECLARED_MODELS: + if model.__tablename__ == table_name: + return model + # This can only go wrong if the migration or model is incorrect + raise + +def fix_geometry_columns(): + update_models = [models.Channel, models.ConnectionNode, models.CrossSectionLocation, + models.Culvert, models.Orifice, models.Pipe, models.Pump, + models.PumpMap, models.Weir, models.Windshielding] + for model in update_models: + op.execute(sa.text(f"SELECT RecoverGeometryColumn('{model.__tablename__}', " + f"'geom', {4326}, '{model.geom.type.geometry_type}', 'XY')")) + op.execute(sa.text(f"SELECT CreateSpatialIndex('{model.__tablename__}', 'geom')")) + + +class Temp(Base): + __tablename__ = f'_temp_228_{uuid.uuid4().hex}' + + id = Column(Integer, primary_key=True) + cross_section_table = Column(String) + cross_section_friction_values = Column(String) + cross_section_vegetation_table = Column(String) + cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) + + +def extend_cross_section_definition_table(): + conn = op.get_bind() + session = Session(bind=op.get_bind()) + # create temporary table + op.execute(sa.text( + f"""CREATE TABLE {Temp.__tablename__} + (id INTEGER PRIMARY KEY, + cross_section_table TEXT, + cross_section_shape INT, + cross_section_width REAL, + cross_section_height REAL, + cross_section_friction_values TEXT, + cross_section_vegetation_table TEXT) + """)) + # copy id's from v2_cross_section_definition + op.execute(sa.text( + f"""INSERT INTO {Temp.__tablename__} (id, cross_section_shape, cross_section_width, cross_section_height) + SELECT id, shape, width, height + FROM v2_cross_section_definition""" + )) + for col_name in ['cross_section_width', 'cross_section_height']: + op.execute(sa.text(f""" + UPDATE {Temp.__tablename__} + SET {col_name} = NULL + WHERE {col_name} = ''; + """)) + + def make_table(*args): + split_args = [arg.split() for arg in args] + if not all(len(args) == len(split_args[0]) for args in split_args): + return + return '\n'.join([','.join(row) for row in zip(*split_args)]) + # Create cross_section_table for tabulated + res = conn.execute(sa.text(f""" + SELECT id, height, width, shape FROM v2_cross_section_definition + WHERE v2_cross_section_definition.shape IN (5,6,7) + AND height IS NOT NULL AND width IS NOT NULL + """)).fetchall() + for id, h, w, s in res: + temp_row = session.query(Temp).filter_by(id=id).first() + # tabulated_YZ: width -> Y; height -> Z + if s == constants.CrossSectionShape.TABULATED_YZ.value: + temp_row.cross_section_table = make_table(w, h) + # tabulated_trapezium or tabulated_rectangle: height, width + else: + temp_row.cross_section_table = make_table(h, w) + session.commit() + # add cross_section_friction_table to cross_section_definition + res = conn.execute(sa.text(""" + SELECT id, friction_values FROM v2_cross_section_definition + WHERE friction_values IS NOT NULL + AND v2_cross_section_definition.shape = 7 + """)).fetchall() + for id, friction_values in res: + temp_row = session.query(Temp).filter_by(id=id).first() + temp_row.cross_section_friction_values = friction_values.replace(' ',',') + session.commit() + # add cross_section_vegetation_table to cross_section_definition + res = conn.execute(sa.text(""" + SELECT id, vegetation_stem_densities, vegetation_stem_diameters, vegetation_heights, vegetation_drag_coefficients + FROM v2_cross_section_definition + WHERE vegetation_stem_densities IS NOT NULL + AND vegetation_stem_diameters IS NOT NULL + AND vegetation_heights IS NOT NULL + AND v2_cross_section_definition.shape = 7 + AND vegetation_drag_coefficients IS NOT NULL + """)).fetchall() + for id, dens, diam, h, c in res: + temp_row = session.query(Temp).filter_by(id=id).first() + temp_row.cross_section_vegetation_table = make_table(dens, diam, h, c) + session.commit() + + +def migrate_cross_section_definition_from_temp(target_table: str, + cols: List[Tuple[str, str]], + def_id_col: str): + for cname, ctype in cols: + op.execute(sa.text(f'ALTER TABLE {target_table} ADD COLUMN {cname} {ctype}')) + # ensure that types work properly + # e.g. heights cannot be text!! + set_query = ','.join( + f'{cname} = (SELECT {cname} FROM {Temp.__tablename__} WHERE id = {target_table}.{def_id_col})' for cname, _ in + cols) + op.execute(sa.text(f""" + UPDATE {target_table} + SET {set_query} + WHERE EXISTS (SELECT 1 FROM {Temp.__tablename__} WHERE id = {target_table}.{def_id_col}); + """)) + op.execute(sa.text(f"UPDATE {target_table} SET cross_section_width = NULL WHERE cross_section_shape IN (5,6,7)")) + op.execute(sa.text(f"UPDATE {target_table} SET cross_section_height = NULL WHERE cross_section_shape IN (5,6,7)")) + + + +def migrate_cross_section_definition_to_location(): + cols = [('cross_section_table', 'TEXT'), + ('cross_section_friction_values', 'TEXT'), + ('cross_section_vegetation_table', 'TEXT'), + ('cross_section_shape', 'INT'), + ('cross_section_width', 'REAL'), + ('cross_section_height', 'REAL')] + migrate_cross_section_definition_from_temp(target_table='v2_cross_section_location', + cols=cols, + def_id_col='definition_id') + +def migrate_cross_section_definition_to_object(table_name: str): + cols = [('cross_section_table', 'TEXT'), + ('cross_section_shape', 'INT'), + ('cross_section_width', 'REAL'), + ('cross_section_height', 'REAL')] + migrate_cross_section_definition_from_temp(target_table=table_name, + cols=cols, + def_id_col='cross_section_definition_id') + + +def set_geom_for_object(table_name: str, col_name: str = 'the_geom'): + # line from connection_node_start_id to connection_node_end_id + # SELECT load_extension('mod_spatialite'); + op.execute(sa.text(f"SELECT AddGeometryColumn('{table_name}', '{col_name}', 4326, 'LINESTRING', 'XY', 0);")) + q = f""" + UPDATE + {table_name} + SET the_geom = ( + SELECT MakeLine(start_node.the_geom, end_node.the_geom) FROM {table_name} AS object + JOIN v2_connection_nodes AS start_node ON object.connection_node_start_id = start_node.id + JOIN v2_connection_nodes AS end_node ON object.connection_node_end_id = end_node.id + ) + """ + op.execute(sa.text(q)) + + +def set_geom_for_v2_pumpstation(): + op.execute(sa.text(f"SELECT AddGeometryColumn('v2_pumpstation', 'the_geom', 4326, 'POINT', 'XY', 0);")) + q = f""" + UPDATE + v2_pumpstation + SET the_geom = ( + SELECT node.the_geom FROM v2_pumpstation AS object + JOIN v2_connection_nodes AS node ON object.connection_node_start_id = node.id + ) + """ + op.execute(sa.text(q)) + + +def get_cols_for_model(model, skip_cols=None): + from sqlalchemy.orm.attributes import InstrumentedAttribute + if skip_cols is None: + skip_cols = [] + return [getattr(model, item) for item in model.__dict__ + if item not in skip_cols + and isinstance(getattr(model, item), InstrumentedAttribute)] + + +def create_sqlite_table_from_model(model): + cols = get_cols_for_model(model, skip_cols = ["id", "geom"]) + op.execute(sa.text(f""" + CREATE TABLE {model.__tablename__} ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + {','.join(f"{col.name} {col.type}" for col in cols)}, + geom {model.geom.type.geometry_type} NOT NULL + );""")) + + +def create_pump_map(): + # Create table + create_sqlite_table_from_model(models.PumpMap) + + # Create geometry + op.execute(sa.text(f"SELECT AddGeometryColumn('v2_pumpstation', 'map_geom', 4326, 'LINESTRING', 'XY', 0);")) + op.execute(sa.text(""" + UPDATE v2_pumpstation + SET map_geom = ( + SELECT MakeLine(start_geom.the_geom, end_geom.the_geom) + FROM v2_connection_nodes AS start_geom, v2_connection_nodes AS end_geom + WHERE v2_pumpstation.connection_node_start_id = start_geom.id + AND v2_pumpstation.connection_node_end_id = end_geom.id + ) + WHERE EXISTS ( + SELECT 1 + FROM v2_connection_nodes AS start_geom, v2_connection_nodes AS end_geom + WHERE v2_pumpstation.connection_node_start_id = start_geom.id + AND v2_pumpstation.connection_node_end_id = end_geom.id + ); + """)) + + # Copy data from v2_pumpstation + new_col_names = ["pump_id", "connection_node_id_end", "code", "display_name", "geom"] + old_col_names = ["id", "connection_node_end_id", "code", "display_name", "map_geom"] + op.execute(sa.text(f""" + INSERT INTO pump_map ({','.join(new_col_names)}) + SELECT {','.join(old_col_names)} FROM v2_pumpstation + WHERE v2_pumpstation.connection_node_end_id IS NOT NULL + AND v2_pumpstation.connection_node_start_id IS NOT NULL + """)) + + +def create_connection_node(): + create_sqlite_table_from_model(models.ConnectionNode) + # copy from v2_connection_nodes + old_col_names = ["id", "initial_waterlevel", "storage_area", "the_geom", "code"] + rename_map = {"initial_waterlevel": "initial_water_level", "the_geom": "geom"} + new_col_names = [rename_map.get(old_name, old_name) for old_name in old_col_names] + op.execute(sa.text(f""" + INSERT INTO connection_node ({','.join(new_col_names)}) + SELECT {','.join(old_col_names)} FROM v2_connection_nodes + """)) + # conditional copy from v2_manhole + old_col_names = ["display_name", "code", "manhole_indicator", + "surface_level", "bottom_level", "drain_level", + "calculation_type", "exchange_thickness", + "hydraulic_conductivity_in", "hydraulic_conductivity_out"] + rename_map = {"surface_level": "manhole_surface_level", + "bottom_level": "bottom_level", + "drain_level": "exchange_level", + "calculation_type": "exchange_type", + "manhole_indicator": "visualisation"} + set_items = ',\n'.join(f"""{rename_map.get(col_name, col_name)} = ( + SELECT v2_manhole.{col_name} FROM v2_manhole + WHERE v2_manhole.connection_node_id = connection_node.id)""" for col_name in old_col_names) + op.execute(sa.text(f""" + UPDATE connection_node + SET {set_items} + WHERE EXISTS ( + SELECT 1 + FROM v2_manhole + WHERE v2_manhole.connection_node_id = connection_node.id + ); + """)) + + +def create_material(): + op.execute(sa.text(""" + CREATE TABLE material ( + id INTEGER PRIMARY KEY NOT NULL, + description TEXT, + friction_type INTEGER, + friction_coefficient REAL); + """)) + session = Session(bind=op.get_bind()) + nof_settings = session.execute(select(func.count()).select_from(models.ModelSettings)).scalar() + if nof_settings > 0: + with open(data_dir.joinpath('0228_materials.csv')) as file: + reader = csv.DictReader(file) + session.bulk_save_objects([models.Material(**row) for row in reader]) + session.commit() + + +def modify_obstacle(): + op.execute(sa.text(f'ALTER TABLE obstacle ADD COLUMN affects_2d BOOLEAN DEFAULT TRUE;')) + op.execute(sa.text(f'ALTER TABLE obstacle ADD COLUMN affects_1d2d_open_water BOOLEAN DEFAULT TRUE;')) + op.execute(sa.text(f'ALTER TABLE obstacle ADD COLUMN affects_1d2d_closed BOOLEAN DEFAULT FALSE;')) + + +def modify_control_target_type(): + for table_name in ['table_control', 'memory_control']: + op.execute(sa.text(f""" + UPDATE {table_name} + SET target_type = REPLACE(target_type, 'v2_pumpstation', 'pump') + WHERE target_type = 'v2_pumpstation'; + """)) + op.execute(sa.text(f""" + UPDATE {table_name} + SET target_type = REPLACE(target_type, 'v2_', '') + WHERE target_type LIKE 'v2_%'; + """)) + + + +def modify_model_settings(): + op.execute(sa.text(f'ALTER TABLE model_settings ADD COLUMN node_open_water_detection INTEGER DEFAULT 1;')) + + +def check_for_null_geoms(): + tables = ["v2_connection_nodes", "v2_cross_section_location", "v2_culvert", "v2_channel", "v2_windshielding"] + conn = op.get_bind() + for table in tables: + nof_null = conn.execute(sa.text(f"SELECT COUNT(*) FROM {table} WHERE the_geom IS NULL;")).fetchone()[0] + if nof_null > 0: + raise Schema228UpgradeException("Cannot migrate because of empty geometries in table {table}") + + +def fix_material_id(): + # Replace migrated material_id's with correct values + replace_map = {9 : 2, 10 : 7} + material_id_tables = ['pipe', 'culvert', 'weir', 'orifice'] + for table in material_id_tables: + op.execute(sa.text(f"UPDATE {table} SET material_id = CASE material_id " + f"{' '.join([f'WHEN {old} THEN {new}' for old, new in replace_map.items()])} " + "ELSE material_id END")) + + + +def drop_conflicting(): + new_tables = [new_name for _, new_name in RENAME_TABLES] + ['material', 'pump_map'] + for table_name in new_tables: + op.execute(f"DROP TABLE IF EXISTS {table_name};") + + + +def upgrade(): + # Empty or non-existing connection node id (start or end) in Orifice, Pipe, Pumpstation or Weir will break + # migration, so an error is raised in these cases + check_for_null_geoms() + # Prevent custom tables in schematisation from breaking migration when they conflict with new table names + drop_conflicting() + # Extent cross section definition table (actually stored in temp) + extend_cross_section_definition_table() + # Migrate data from cross_section_definition to cross_section_location + migrate_cross_section_definition_to_location() + # Prepare object tables for renaming by copying cross section data and setting the_geom + for table_name in ['v2_culvert', 'v2_weir', 'v2_pipe', 'v2_orifice']: + migrate_cross_section_definition_to_object(table_name) + # Set geometry for tables without one + if table_name != 'v2_culvert': + set_geom_for_object(table_name) + set_geom_for_v2_pumpstation() + for old_table_name, new_table_name in RENAME_TABLES: + modify_table(old_table_name, new_table_name) + # Create new tables + create_pump_map() + create_material() + create_connection_node() + # Modify exsiting tables + modify_model_settings() + modify_obstacle() + modify_control_target_type() + fix_material_id() + fix_geometry_columns() + remove_tables([old for old, _ in RENAME_TABLES]+DELETE_TABLES+[Temp.__tablename__, 'v2_manhole']) + + +def downgrade(): + # Not implemented on purpose + raise NotImplementedError("Downgrade back from 0.3xx is not supported") diff --git a/threedi_schema/migrations/versions/data/0228_materials.csv b/threedi_schema/migrations/versions/data/0228_materials.csv new file mode 100644 index 00000000..150ada61 --- /dev/null +++ b/threedi_schema/migrations/versions/data/0228_materials.csv @@ -0,0 +1,10 @@ +id,description,friction_type,friction_coefficient +0,Concrete,2,0.0145 +1,PVC,2,0.011 +2,Gres,2,0.0115 +3,Cast iron,2,0.0135 +4,Brickwork,2,0.016 +5,HPE,2,0.011 +6,HDPE,2,0.011 +7,Plate iron,2,0.0135 +8,Steel,2,0.013 diff --git a/threedi_schema/scripts.py b/threedi_schema/scripts.py index a251b6a4..98b82a68 100644 --- a/threedi_schema/scripts.py +++ b/threedi_schema/scripts.py @@ -41,7 +41,6 @@ def migrate( schema.upgrade( revision=revision, backup=backup, - set_views=set_views, upgrade_spatialite_version=upgrade_spatialite_version, convert_to_geopackage=convert_to_geopackage, ) diff --git a/threedi_schema/tests/conftest.py b/threedi_schema/tests/conftest.py index cb9ae0db..477181a0 100644 --- a/threedi_schema/tests/conftest.py +++ b/threedi_schema/tests/conftest.py @@ -59,5 +59,5 @@ def in_memory_sqlite(): def sqlite_latest(in_memory_sqlite): """An in-memory database with the latest schema version""" db = ThreediDatabase("") - in_memory_sqlite.schema.upgrade("head", backup=False, set_views=False) + in_memory_sqlite.schema.upgrade("head", backup=False) return db diff --git a/threedi_schema/tests/test_migration.py b/threedi_schema/tests/test_migration.py index af69e5d4..6581f43d 100644 --- a/threedi_schema/tests/test_migration.py +++ b/threedi_schema/tests/test_migration.py @@ -89,6 +89,49 @@ def test_upgrade_success(sqlite_file, tmp_path_factory): pytest.fail(f"Failed to upgrade {sqlite_file}") +class TestMigration228: + pytestmark = pytest.mark.migration_228 + removed_tables = set(["v2_channel", + "v2_windshielding", + "v2_cross_section_location", + "v2_pipe", + "v2_culvert", + "v2_weir", + "v2_orifice", + "v2_pumpstation", + "v2_cross_section_definition", + "v2_floodfill", + "v2_connection_nodes"]) + added_tables = set(["channel", + "windshielding_1d", + "cross_section_location", + "pipe", + "culvert", + "weir", + "orifice", + "pump", + "connection_node", + "material", + "pump_map"]) + + def test_tables(self, schema_ref, schema_upgraded): + # Test whether the added tables are present + # and whether the removed tables are not present* + tables_new = set(get_sql_tables(get_cursor_for_schema(schema_upgraded))) + assert self.added_tables.issubset(tables_new) + assert self.removed_tables.isdisjoint(tables_new) + + + def test_columns_added_tables(self, schema_upgraded): + # Note that only the added tables are touched. + # So this check covers both added and removed columns. + cursor = get_cursor_for_schema(schema_upgraded) + for table in self.added_tables: + cols_sqlite = get_columns_from_sqlite(cursor, table) + cols_schema = get_columns_from_schema(schema_upgraded, table) + assert cols_sqlite == cols_schema + + class TestMigration226: pytestmark = pytest.mark.migration_226 removed_tables = set(['v2_dem_average_area', diff --git a/threedi_schema/tests/test_migration_213.py b/threedi_schema/tests/test_migration_213.py index a862ea68..0b7f0f04 100644 --- a/threedi_schema/tests/test_migration_213.py +++ b/threedi_schema/tests/test_migration_213.py @@ -24,7 +24,7 @@ def sqlite_v212(): """An in-memory database with schema version 212""" db = ThreediDatabase("") - ModelSchema(db).upgrade("0212", backup=False, set_views=False) + ModelSchema(db).upgrade("0212", backup=False) return db diff --git a/threedi_schema/tests/test_schema.py b/threedi_schema/tests/test_schema.py index bffdb269..5e243679 100644 --- a/threedi_schema/tests/test_schema.py +++ b/threedi_schema/tests/test_schema.py @@ -7,7 +7,6 @@ from threedi_schema.application import errors from threedi_schema.application.schema import get_schema_version from threedi_schema.domain import constants -from threedi_schema.domain.views import ALL_VIEWS from threedi_schema.infrastructure.spatialite_versions import get_spatialite_version @@ -113,17 +112,17 @@ def test_validate_schema_too_high_migration(sqlite_latest, version): def test_full_upgrade_empty(in_memory_sqlite): """Upgrade an empty database to the latest version""" schema = ModelSchema(in_memory_sqlite) - schema.upgrade(backup=False, set_views=False, upgrade_spatialite_version=False) + schema.upgrade(backup=False, upgrade_spatialite_version=False) assert schema.get_version() == get_schema_version() - assert in_memory_sqlite.has_table("v2_connection_nodes") + assert in_memory_sqlite.has_table("connection_node") def test_full_upgrade_with_preexisting_version(south_latest_sqlite): """Upgrade an empty database to the latest version""" schema = ModelSchema(south_latest_sqlite) - schema.upgrade(backup=False, set_views=False, upgrade_spatialite_version=False) + schema.upgrade(backup=False, upgrade_spatialite_version=False) assert schema.get_version() == get_schema_version() - assert south_latest_sqlite.has_table("v2_connection_nodes") + assert south_latest_sqlite.has_table("connection_node") # https://github.com/nens/threedi-schema/issues/10: assert not south_latest_sqlite.has_table("v2_levee") @@ -131,9 +130,9 @@ def test_full_upgrade_with_preexisting_version(south_latest_sqlite): def test_full_upgrade_oldest(oldest_sqlite): """Upgrade a legacy database to the latest version""" schema = ModelSchema(oldest_sqlite) - schema.upgrade(backup=False, set_views=False, upgrade_spatialite_version=False) + schema.upgrade(backup=False, upgrade_spatialite_version=False) assert schema.get_version() == get_schema_version() - assert oldest_sqlite.has_table("v2_connection_nodes") + assert oldest_sqlite.has_table("connection_node") # https://github.com/nens/threedi-schema/issues/10: assert not oldest_sqlite.has_table("v2_levee") @@ -145,9 +144,7 @@ def test_upgrade_south_not_latest_errors(in_memory_sqlite): schema, "get_version", return_value=constants.LATEST_SOUTH_MIGRATION_ID - 1 ): with pytest.raises(errors.MigrationMissingError): - schema.upgrade( - backup=False, set_views=False, upgrade_spatialite_version=False - ) + schema.upgrade(backup=False, upgrade_spatialite_version=False) def test_upgrade_with_backup(south_latest_sqlite): @@ -157,9 +154,7 @@ def test_upgrade_with_backup(south_latest_sqlite): "threedi_schema.application.schema._upgrade_database", side_effect=RuntimeError ) as upgrade, mock.patch.object(schema, "get_version", return_value=199): with pytest.raises(RuntimeError): - schema.upgrade( - backup=True, set_views=False, upgrade_spatialite_version=False - ) + schema.upgrade(backup=True, upgrade_spatialite_version=False) (db,), kwargs = upgrade.call_args assert db is not south_latest_sqlite @@ -172,55 +167,19 @@ def test_upgrade_without_backup(south_latest_sqlite): "threedi_schema.application.schema._upgrade_database", side_effect=RuntimeError ) as upgrade, mock.patch.object(schema, "get_version", return_value=199): with pytest.raises(RuntimeError): - schema.upgrade( - backup=False, set_views=False, upgrade_spatialite_version=False - ) + schema.upgrade(backup=False, upgrade_spatialite_version=False) (db,), kwargs = upgrade.call_args assert db is south_latest_sqlite -@pytest.mark.parametrize( - "set_views, upgrade_spatialite_version", - [ - (True, False), - (False, True), - (True, True), - ], -) -@pytest.mark.filterwarnings("ignore::UserWarning") -def test_set_views(oldest_sqlite, set_views, upgrade_spatialite_version): - """Make sure that the views are regenerated""" - schema = ModelSchema(oldest_sqlite) - schema.upgrade( - backup=False, - set_views=set_views, - upgrade_spatialite_version=upgrade_spatialite_version, - ) - assert schema.get_version() == get_schema_version() - - # Test all views - with oldest_sqlite.session_scope() as session: - for view_name in ALL_VIEWS: - session.execute(text(f"SELECT * FROM {view_name} LIMIT 1")).fetchall() - - -def test_set_views_warning(oldest_sqlite): - schema = ModelSchema(oldest_sqlite) - with pytest.warns(UserWarning): - schema.upgrade(backup=False, set_views=False, upgrade_spatialite_version=True) - - def test_convert_to_geopackage_raise(oldest_sqlite): if get_schema_version() >= 300: pytest.skip("Warning not expected beyond schema 300") schema = ModelSchema(oldest_sqlite) with pytest.raises(errors.UpgradeFailedError): schema.upgrade( - backup=False, - set_views=False, - upgrade_spatialite_version=False, - convert_to_geopackage=True, + backup=False, upgrade_spatialite_version=False, convert_to_geopackage=True ) @@ -244,7 +203,7 @@ def test_upgrade_spatialite_3(oldest_sqlite): # the spatial indexes are there with oldest_sqlite.engine.connect() as connection: check_result = connection.execute( - text("SELECT CheckSpatialIndex('v2_connection_nodes', 'the_geom')") + text("SELECT CheckSpatialIndex('connection_node', 'geom')") ).scalar() assert check_result == 1 @@ -258,7 +217,7 @@ def test_set_spatial_indexes(in_memory_sqlite): with engine.connect() as connection: with connection.begin(): connection.execute( - text("SELECT DisableSpatialIndex('v2_connection_nodes', 'the_geom')") + text("SELECT DisableSpatialIndex('connection_node', 'geom')") ).scalar() connection.execute(text("DROP TABLE idx_v2_connection_nodes_the_geom")) @@ -266,7 +225,7 @@ def test_set_spatial_indexes(in_memory_sqlite): with engine.connect() as connection: check_result = connection.execute( - text("SELECT CheckSpatialIndex('v2_connection_nodes', 'the_geom')") + text("SELECT CheckSpatialIndex('connection_node', 'geom')") ).scalar() assert check_result == 1 diff --git a/threedi_schema/tests/test_spatalite_versions.py b/threedi_schema/tests/test_spatalite_versions.py index 40abab89..d1696a0e 100644 --- a/threedi_schema/tests/test_spatalite_versions.py +++ b/threedi_schema/tests/test_spatalite_versions.py @@ -1,7 +1,6 @@ from sqlalchemy import Column, func, Integer, String from sqlalchemy.orm import declarative_base -from threedi_schema.domain import models from threedi_schema.domain.custom_types import Geometry from threedi_schema.infrastructure.spatialite_versions import ( copy_model, @@ -39,15 +38,15 @@ def test_copy_model(empty_sqlite_v3, empty_sqlite_v4): session.commit() # Copy it - copy_model(db_from, db_to, models.ConnectionNode) + copy_model(db_from, db_to, TestModel) # Check if it is present in 'db_to' with db_to.session_scope() as session: records = list( session.query( - models.ConnectionNode.id, - models.ConnectionNode.code, - func.ST_AsText(models.ConnectionNode.the_geom), + TestModel.id, + TestModel.code, + func.ST_AsText(TestModel.the_geom), ) ) diff --git a/threedi_schema/tests/test_upgrade_utils.py b/threedi_schema/tests/test_upgrade_utils.py new file mode 100644 index 00000000..21463b70 --- /dev/null +++ b/threedi_schema/tests/test_upgrade_utils.py @@ -0,0 +1,56 @@ +import logging +from pathlib import Path +from unittest.mock import call, MagicMock + +import pytest + +from threedi_schema.application import upgrade_utils +from threedi_schema.application.schema import get_alembic_config +from threedi_schema.application.threedi_database import ThreediDatabase + +data_dir = Path(__file__).parent / "data" + + +def test_progress_handler(): + progress_func = MagicMock() + mock_record = MagicMock(levelno=logging.INFO, percent=40) + expected_calls = [call(100 * i / 5) for i in range(5)] + handler = upgrade_utils.ProgressHandler(progress_func, total_steps=5) + for _ in range(5): + handler.handle(mock_record) + assert progress_func.call_args_list == expected_calls + + +@pytest.mark.parametrize( + "target_revision, nsteps_expected", [("0226", 5), ("0200", 0), (None, 0)] +) +def test_get_upgrade_steps_count(target_revision, nsteps_expected): + schema = ThreediDatabase(data_dir.joinpath("v2_bergermeer_221.sqlite")).schema + nsteps = upgrade_utils.get_upgrade_steps_count( + config=get_alembic_config(), + current_revision=schema.get_version(), + target_revision=target_revision, + ) + assert nsteps == nsteps_expected + + +def test_get_upgrade_steps_count_pre_200(oldest_sqlite): + schema = oldest_sqlite.schema + nsteps = upgrade_utils.get_upgrade_steps_count( + config=get_alembic_config(), + current_revision=schema.get_version(), + target_revision="0226", + ) + assert nsteps == 27 + + +def test_upgrade_with_progress_func(oldest_sqlite): + schema = oldest_sqlite.schema + progress_func = MagicMock() + schema.upgrade( + backup=False, + upgrade_spatialite_version=False, + progress_func=progress_func, + revision="0201", + ) + assert progress_func.call_args_list == [call(0.0), call(50.0)]