Skip to content

Commit

Permalink
Import swmminp refactor (#1543)
Browse files Browse the repository at this point in the history 
* Parser added
Coordinate check
Inflows added

* Patterns added

* structures the function

* Finished the time series

* Finished the curves

* Inlets junctions ready and the code to import from INP

* Outfalls added

* Storage Units added

* Conduits added

* Orifices, pumps and weirs added

* Fixed weir xsections and improved code for import SWMM INP.

* Fixed orifices xsections

* Improved and fixed the export code for the patterns

* Added messages of how many features were added from the INP

* Added the logic to check for existing SD

* Finished the Keep existing and complete for ORIFICES

* Finished the Keep existing and complete for WEIRS

* Finished the Keep existing and complete for PUMPS

* Finished the Keep existing and complete for CONDUITS

* Finished the Keep existing and complete for STORAGE and improve the code by using coordinates dict

* Finished the Keep existing and complete for OUTFALLS and improve the code by using coordinates dict

* Finished the Keep existing and complete for JUNCTIONS and improve the code by using coordinates dict

* Finished the Keep existing and complete for CURVES

* Finished the Keep existing and complete for TIMESERIES

* Finished the Keep existing and complete for INFLOWS

* Finished the Keep existing and complete for PATTERNS

* Removed the CONDUITS outside the domain

* Removed the PUMPS outside the domain

* Removed the ORIFICES outside the domain

* Removed the WEIRS outside the domain

* Removed the STORAGES JUNCTIONS and OUTFALLS outside the domain

* Removed the necessity of TOPO.DAT when importing SWMM.INP

* Allowed for different INP names when importing from INP

* Added the new system to the import from .DAT

* Added the new system to the import from components - single component

* Added the new system to the import from components - several components

* Removed some commented out portions of the code

* Added the import dropbox and sdclogging for the several components

* Check the test

* SelfHelpKit test basic data

* Removed the rt and culvert from testing. Check later

* Added back the swmm related tests

* Removed sd_clogging

* Adjusted the testing data

* Adjusted the geopackage testing data

* Adjusted the coastal storm drain testing data

* Fixed the MUL.DAT data

* Fixed the import_mult

* Fixed the MULT.DAT for the MULTCHAN project and the data_2

* Fixed the mult parser

* Fixed the import_2 mult parser

* Added the SWMM.INP back to the Coastal and small fixes

* Imported the swmminp to run the test on swmmflort

* Added back the rating table and culverts for the self help kit

* Finished the testing for storm drain

* Fixed the metadata

* Convert the fixed stage * to 0
  • Loading branch information
@rpachaly
rpachaly authored Sep 6, 2024
1 parent a16a509 commit df08ac8
Show file tree
Hide file tree
Showing 17 changed files with 3,821 additions and 458 deletions.
171 changes: 108 additions & 63 deletions flo2d/flo2d.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@
import pip
from qgis.PyQt import QtCore, QtGui
from PyQt5.QtGui import QFont
from PyQt5.QtWidgets import QApplication, QToolButton, QProgressDialog, QDockWidget, QTabWidget, QWidget, QVBoxLayout
from PyQt5.QtWidgets import QApplication, QToolButton, QProgressDialog, QDockWidget, QTabWidget, QWidget, QVBoxLayout, \
QPushButton
from osgeo import gdal, ogr
from qgis._core import QgsMessageLog, QgsCoordinateReferenceSystem, QgsMapSettings, QgsProjectMetadata, \
QgsMapRendererParallelJob, QgsLayerTreeLayer, QgsVectorLayerExporter, QgsVectorFileWriter, QgsVectorLayer, \
Expand Down Expand Up @@ -1834,6 +1835,8 @@ def call_IO_methods_dat(self, calls, debug, *args):
dat = "BRIDGE_COEFF_DATA.DAT"
elif call == "import_hystruc_bridge_xs":
dat = "BRIDGE_XSEC.DAT"
elif call == "import_swmminp":
dat = "SWMM.INP"
else:
dat = call.split("_")[-1].upper() + ".DAT"
if call.startswith("import"):
Expand Down Expand Up @@ -1924,6 +1927,7 @@ def import_gds(self):
"import_breach",
"import_gutter",
"import_fpfroude",
"import_swmminp",
"import_swmmflo",
"import_swmmflort",
"import_swmmoutf",
Expand Down Expand Up @@ -2043,6 +2047,7 @@ def import_gds(self):
import_calls.remove("import_raincell")

if "Storm Drain" not in dlg_components.components:
import_calls.remove("import_swmminp")
import_calls.remove("import_swmmflo")
import_calls.remove("import_swmmflort")
import_calls.remove("import_swmmoutf")
Expand Down Expand Up @@ -2195,30 +2200,6 @@ def import_gds(self):
for table in tables:
self.gutils.clear_tables(table)

# Import first the grid
if "import_cont_toler" in import_calls:
self.call_IO_methods(["import_cont_toler"], True)
import_calls.remove("import_cont_toler")

if "import_mannings_n_topo" in import_calls:
self.call_IO_methods(["import_mannings_n_topo"], True)
import_calls.remove("import_mannings_n_topo")

# Import the SWMM.INP
if "Storm Drain" in dlg_components.components:
swmm_converter = SchemaSWMMConverter(self.con, self.iface, self.lyrs)
swmm_converter.create_user_swmm_inlets_junctions()
swmm_converter.create_user_swmm_outlets()

if os.path.isfile(dir_name + r"\SWMM.INP"):
if self.f2d_widget.storm_drain_editor.import_storm_drain_INP_file(
"Force import of SWMM.INP", False
):
self.files_used += "SWMM.INP" + "\n"
else:
self.uc.bar_error("ERROR 100623.0944: SWMM.INP file not found!")
self.uc.log_info("ERROR 100623.0944: SWMM.INP file not found!")

self.call_IO_methods(import_calls, True) # The strings list 'export_calls', contains the names of
# the methods in the class Flo2dGeoPackage to import (read) the
# FLO-2D .DAT files
Expand Down Expand Up @@ -2636,6 +2617,7 @@ def import_selected_components(self):
"import_breach",
"import_gutter",
"import_fpfroude",
"import_swmminp",
"import_swmmflo",
"import_swmmflort",
"import_swmmoutf",
Expand Down Expand Up @@ -2729,9 +2711,12 @@ def import_selected_components(self):
import_calls.remove("import_raincell")

if "Storm Drain" not in dlg_components.components:
import_calls.remove("import_swmminp")
import_calls.remove("import_swmmflo")
import_calls.remove("import_swmmflort")
import_calls.remove("import_swmmoutf")
import_calls.remove("import_swmmflodropbox")
import_calls.remove("import_sdclogging")

if "Spatial Tolerance" not in dlg_components.components:
import_calls.remove("import_tolspatial")
Expand All @@ -2741,29 +2726,6 @@ def import_selected_components(self):

if import_calls:

if "Storm Drain" in dlg_components.components:
try:
swmm_converter = SchemaSWMMConverter(self.con, self.iface, self.lyrs)
swmm_converter.create_user_swmm_inlets_junctions()
swmm_converter.create_user_swmm_outlets()
except Exception as e:
self.uc.log_info(traceback.format_exc())
QApplication.restoreOverrideCursor()
self.uc.show_error(
"ERROR 100623.1044:\n\nConverting Schematic SD Inlets to User Storm Drain Inlets/Junctions failed!"
+ "\n_______________________________________________________________",
e,
)

if os.path.isfile(outdir + r"\SWMM.INP"):
if self.f2d_widget.storm_drain_editor.import_storm_drain_INP_file(
"Force import of SWMM.INP", True
):
self.files_used += "SWMM.INP" + "\n"
else:
self.uc.bar_error("ERROR 100623.0944: SWMM.INP file not found!")
self.uc.log_info("ERROR 100623.0944: SWMM.INP file not found!")

self.call_IO_methods(
import_calls, True
) # The strings list 'import_calls', contains the names of
Expand Down Expand Up @@ -2845,6 +2807,25 @@ def import_selected_components2(self):
"""
self.gutils.disable_geom_triggers()
self.f2g = Flo2dGeoPackage(self.con, self.iface)
s = QSettings()
last_dir = s.value("FLO-2D/lastGdsDir", "")
fname, __ = QFileDialog.getOpenFileName(
None, "Select FLO-2D file to import", directory=last_dir, filter="DAT or INP (*.DAT *.dat *.INP *.inp)"
)
if not fname:
self.gutils.enable_geom_triggers()
return
dir_name = os.path.dirname(fname)
s.setValue("FLO-2D/lastGdsDir", dir_name)
bname = os.path.basename(fname)

if bname.lower().endswith("inp"):
swmm_file_name = bname
swmm_file_path = fname
else:
swmm_file_name = "SWMM.INP"
swmm_file_path = os.path.join(dir_name, swmm_file_name)

file_to_import_calls = {
"CONT.DAT": "import_cont_toler",
"TOLER.DAT": "import_cont_toler",
Expand Down Expand Up @@ -2872,6 +2853,7 @@ def import_selected_components2(self):
"BREACH.DAT": "import_breach",
"GUTTER.DAT": "import_gutter",
"FPFROUDE.DAT": "import_fpfroude",
f"{swmm_file_name}": "import_swmminp",
"SWMMFLO.DAT": "import_swmmflo",
"SWMMFLORT.DAT": "import_swmmflort",
"SWMMOUTF.DAT": "import_swmmoutf",
Expand All @@ -2882,22 +2864,11 @@ def import_selected_components2(self):
"MANNINGS_N.DAT": "import_mannings_n",
"TOPO.DAT": "import_topo"
}
s = QSettings()
last_dir = s.value("FLO-2D/lastGdsDir", "")
fname, __ = QFileDialog.getOpenFileName(
None, "Select FLO-2D file to import", directory=last_dir, filter="(*.DAT)"
)
if not fname:
self.gutils.enable_geom_triggers()
return
dir_name = os.path.dirname(fname)
s.setValue("FLO-2D/lastGdsDir", dir_name)
bname = os.path.basename(fname)

if bname not in file_to_import_calls:
QMessageBox.critical(
self.iface.mainWindow(),
"Import selected GDS file",
"Import selected DAT file",
"Import from {0} file is not supported.".format(bname),
)
self.uc.log_info(f"Import from {bname} file is not supported.")
Expand All @@ -2909,11 +2880,14 @@ def import_selected_components2(self):
QApplication.setOverrideCursor(Qt.WaitCursor)
try:
method = getattr(self.f2g, call_string)
method()
if call_string == "import_swmminp":
method(swmm_file=swmm_file_path)
else:
method()
QApplication.restoreOverrideCursor()
QMessageBox.information(
self.iface.mainWindow(),
"Import selected GDS file",
"Import selected DAT file",
"Import from {0} was successful.".format(bname),
)
self.uc.log_info(f"Import from {bname} was successful.")
Expand Down Expand Up @@ -3439,7 +3413,78 @@ def import_inp(self):
"""
Function to export FLO-2D to SWMM's INP file
"""
self.f2d_widget.storm_drain_editor.import_storm_drain_INP_file("Choose", True)
try:

if self.gutils.is_table_empty("grid"):
self.uc.bar_warn("There is no grid! Please create it before running tool.")
self.uc.log_info("There is no grid! Please create it before running tool.")
return False

QApplication.setOverrideCursor(Qt.WaitCursor)

self.f2g = Flo2dGeoPackage(self.con, self.iface, parsed_format="DAT")
s = QSettings()
last_dir = s.value("FLO-2D/lastGdsDir", "")
fname, __ = QFileDialog.getOpenFileName(
None, "Select SWMM INP file to import", directory=last_dir, filter="(*.INP)"
)
if not fname:
QApplication.restoreOverrideCursor()
return

dir_name = os.path.dirname(fname)
s.setValue("FLO-2D/lastGdsDir", dir_name)

sd_user_tables = [
'user_swmm_inlets_junctions',
'user_swmm_conduits',
'user_swmm_pumps',
'user_swmm_orifices',
'user_swmm_weirs',
'user_swmm_outlets',
'user_swmm_storage_units'
]
empty_sd = all((self.gutils.is_table_empty(sd_user_table) for sd_user_table in sd_user_tables))

if self.f2g.set_parser(fname, get_cell_size=False):
if not empty_sd:
QApplication.restoreOverrideCursor()
msg = QMessageBox()
msg.setWindowTitle("Replace or complete Storm Drain User Data")
msg.setText(
"There is already Storm Drain data in the Users Layers.\n\nWould you like to keep it and "
"complete it with data taken from the .INP file?\n\n"
+ "or you prefer to erase it and create new storm drains from the .INP file?\n"
)

msg.addButton(QPushButton("Keep existing and complete"), QMessageBox.YesRole)
msg.addButton(QPushButton("Create new Storm Drains"), QMessageBox.NoRole)
msg.addButton(QPushButton("Cancel"), QMessageBox.RejectRole)
msg.setDefaultButton(QMessageBox().Cancel)
msg.setIcon(QMessageBox.Question)
ret = msg.exec_()
QApplication.setOverrideCursor(Qt.WaitCursor)
if ret == 0:
self.f2g.import_swmminp(swmm_file=fname, delete_existing=False)
elif ret == 1:
self.f2g.import_swmminp(swmm_file=fname)
else:
QApplication.restoreOverrideCursor()
return
else:
self.f2g.import_swmminp(swmm_file=fname)

self.lyrs.refresh_layers()

self.uc.bar_info("Import from INP completed! Check log messages for more information. ")
self.uc.log_info("Import from INP completed!")

QApplication.restoreOverrideCursor()

except Exception as e:
QApplication.restoreOverrideCursor()
self.uc.log_info(f"ERROR 08272024.0932: Could not import SWMM INP file!\n{e}")
self.uc.bar_error("ERROR 08272024.0932: Could not import SWMM INP file!")

@connection_required
def export_inp(self):
Expand Down
47 changes: 45 additions & 2 deletions flo2d/flo2d_ie/flo2d_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version
import os
import re
from collections import OrderedDict, defaultdict
from itertools import chain, repeat, zip_longest
from operator import attrgetter
Expand Down Expand Up @@ -312,6 +313,7 @@ def __init__(self):
"FPXSEC.DAT": None,
"BREACH.DAT": None,
"FPFROUDE.DAT": None,
"SWMM.INP": None,
"SWMMFLO.DAT": None,
"SWMMFLORT.DAT": None,
"SWMMOUTF.DAT": None,
Expand Down Expand Up @@ -416,6 +418,41 @@ def double_parser(file1, file2):
if row:
yield row

@staticmethod
def swmminp_parser(swmminp_file):
"""
This is the new swmm parser. Usage example:
To get data from a specific section
junctions_data = sections.get('JUNCTIONS', [])
To print all sections
for section, lines in sections.items():
print(f"[{section}]")
for line in lines:
print(line)
print("\n")
"""
sections = defaultdict(list)
current_section = None

with open(swmminp_file, 'r') as inp_file:
for line in inp_file:
line = line.strip()

# Ignore empty lines and comments
if not line or line.startswith(';'):
continue

# Check for section headers (e.g., [JUNCTIONS])
if line.startswith('[') and line.endswith(']'):
current_section = line[1:-1].strip().upper() # Strip brackets and set as current section
elif current_section:
# Split the line by any whitespace (e.g., space, tab)
sections[current_section].append(re.split(r'\s+', line))

return sections

@staticmethod
def fix_row_size(row, fix_size, default=None, index=None):
loops = fix_size - len(row)
Expand Down Expand Up @@ -840,10 +877,8 @@ def parse_mult(self):
mult = self.dat_files["MULT.DAT"]
par = self.single_parser(mult)
head = next(par)
self.fix_row_size(head, 8)
data = []
for row in par:
self.fix_row_size(row, 5)
data.append(row)
return head, data

Expand Down Expand Up @@ -963,6 +998,14 @@ def parse_gutter(self):
data.append(row)
return head, data

def parse_swmminp(self, swmm_file):
if swmm_file == "SWMM.INP":
swmminp = self.dat_files[swmm_file]
else:
swmminp = swmm_file
swmminp_dict = self.swmminp_parser(swmminp)
return swmminp_dict

def parse_swmmflo(self):
swmmflo = self.dat_files["SWMMFLO.DAT"]
par = self.single_parser(swmmflo)
Expand Down
Loading

0 comments on commit df08ac8

Please sign in to comment.