From b1c454fd761b78788975a1f6ee301c1999d685a2 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 22 Oct 2020 17:23:42 +0200 Subject: [PATCH 01/35] [feature] add a datamodel init tool to create or update a QGEP database --- qgepplugin/datamodel_initializer/__init__.py | 178 ++++++++++ qgepplugin/gui/qgepdatamodeldialog.py | 295 +++++++++++++++ qgepplugin/qgepplugin.py | 11 + qgepplugin/ui/qgepdatamodeldialog.ui | 355 +++++++++++++++++++ 4 files changed, 839 insertions(+) create mode 100644 qgepplugin/datamodel_initializer/__init__.py create mode 100644 qgepplugin/gui/qgepdatamodeldialog.py create mode 100644 qgepplugin/ui/qgepdatamodeldialog.ui diff --git a/qgepplugin/datamodel_initializer/__init__.py b/qgepplugin/datamodel_initializer/__init__.py new file mode 100644 index 00000000..61439c43 --- /dev/null +++ b/qgepplugin/datamodel_initializer/__init__.py @@ -0,0 +1,178 @@ +import os +import tempfile +import hashlib +import zipfile +import subprocess +import pkg_resources +import site +import importlib + +from qgis.PyQt.QtCore import QUrl, QFile, QIODevice +from qgis.PyQt.QtNetwork import QNetworkRequest +from qgis.core import QgsNetworkAccessManager, QgsFeedback, QgsMessageLog, Qgis, QgsProject + + +# Basic config +DATAMODEL_RELEASE = '1.5.2' +QGEP_RELEASE = '8.0' + +# Derived urls/paths, may require adaptations if release structure changes +DATAMODEL_RELEASE_URL = f'https://github.com/QGEP/datamodel/archive/v{DATAMODEL_RELEASE}.zip' +QGEP_RELEASE_URL = f'https://github.com/QGEP/QGEP/releases/download/v{QGEP_RELEASE}/qgep.zip' +hash = hashlib.md5((DATAMODEL_RELEASE_URL + QGEP_RELEASE_URL).encode('utf-8')).hexdigest()[0:8] +RELEASE_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', hash) +QGIS_PROJECT_PATH = os.path.join(RELEASE_DIR, 'project', 'qgep.qgs') +DATAMODEL_DIR = os.path.join(RELEASE_DIR, f'datamodel-{DATAMODEL_RELEASE}') +REQUIREMENTS_PATH = os.path.join(DATAMODEL_DIR, 'requirements.txt') +DATAMODEL_DELTAS_DIR = os.path.join(DATAMODEL_DIR, "delta") + + +class QGEPDatamodelError(Exception): + pass + + +def _run_cmd(shell_command, cwd=None, error_message='Subprocess error, see logs for more information'): + """ + Helper to run commands through subprocess + """ + QgsMessageLog.logMessage(f"Running command : {shell_command}", "QGEP") + result = subprocess.run(shell_command, cwd=cwd, shell=True, capture_output=True) + if result.stdout: + QgsMessageLog.logMessage(result.stdout.decode(), "QGEP") + if result.stderr: + QgsMessageLog.logMessage(result.stderr.decode(), "QGEP", level=Qgis.Critical) + if result.returncode: + raise QGEPDatamodelError(error_message) + return result.stdout.decode() + + +def check_release_exists(): + return os.path.exists(QGIS_PROJECT_PATH) and os.path.exists(DATAMODEL_DIR) + + +def check_python_requirements(): + return len(missing_python_requirements()) == 0 + + +def missing_python_requirements(): + # see https://stackoverflow.com/a/45474387/13690651 + + missing = [] + if not os.path.exists(REQUIREMENTS_PATH): + missing.append(('unknown', 'requirements not found')) + else: + requirements = pkg_resources.parse_requirements(open(REQUIREMENTS_PATH)) + for requirement in requirements: + try: + pkg_resources.require(str(requirement)) + except pkg_resources.DistributionNotFound: + missing.append((requirement, 'missing')) + except pkg_resources.VersionConflict: + missing.append((requirement, 'conflict')) + return missing + + +def install_deps(): + + network_manager = QgsNetworkAccessManager.instance() + feedback = QgsFeedback() + + # Download the files if needed + + if check_release_exists(): + QgsMessageLog.logMessage(f"Required files are already present in {RELEASE_DIR}", "QGEP") + + else: + os.makedirs(RELEASE_DIR, exist_ok=True) + + # Download datamodel + QgsMessageLog.logMessage(f"Downloading {DATAMODEL_RELEASE_URL} to {RELEASE_DIR}", "QGEP") + reply = network_manager.blockingGet(QNetworkRequest(QUrl(DATAMODEL_RELEASE_URL)), feedback=feedback) + datamodel_file = QFile(os.path.join(RELEASE_DIR, 'datamodel.zip')) + datamodel_file.open(QIODevice.WriteOnly) + datamodel_file.write(reply.content()) + datamodel_file.close() + + # Download QGEP + QgsMessageLog.logMessage(f"Downloading {QGEP_RELEASE_URL} to {RELEASE_DIR}", "QGEP") + reply = network_manager.blockingGet(QNetworkRequest(QUrl(QGEP_RELEASE_URL)), feedback=feedback) + qgep_file = QFile(os.path.join(RELEASE_DIR, 'qgep.zip')) + qgep_file.open(QIODevice.WriteOnly) + qgep_file.write(reply.content()) + qgep_file.close() + + QgsMessageLog.logMessage(f"Extracting files to {RELEASE_DIR}", "QGEP") + + # Unzip datamodel + datamodel_zip = zipfile.ZipFile(datamodel_file.fileName()) + datamodel_zip.extractall(RELEASE_DIR) + + # Unzip QGEP + qgep_zip = zipfile.ZipFile(qgep_file.fileName()) + qgep_zip.extractall(RELEASE_DIR) + + # TODO : Ideally, this should be done in a venv, as to avoid permission issues and/or modification + # of libraries versions that could affect other parts of the system. + # We could initialize a venv in the user's directory, and activate it. + # It's almost doable when only running commands from the command line (in which case we could + # just prepent something like `path/to/venv/Scripts/activate && ` to commands, /!\ syntax differs on Windows), + # but to be really useful, it would be best to then enable the virtualenv from within python directly. + # It seems venv doesn't provide a way to do so, while virtualenv does + # (see https://stackoverflow.com/a/33637378/13690651) + # but virtualenv isn't in the stdlib... So we'd have to install it globally ! Argh... + # Anyway, pip deps support should be done in QGIS one day so all plugins can benefit. + # In the mean time we just install globally and hope for the best. + + # Install dependencies + QgsMessageLog.logMessage(f"Installing python dependencies from {REQUIREMENTS_PATH}", "QGEP") + _run_cmd(f'pip install -r {REQUIREMENTS_PATH}', error_message='Could not install python dependencies') + + # Refresh paths + importlib.reload(site) + + +def get_current_version(): + if not os.path.exists(DATAMODEL_DELTAS_DIR): + return None + + pum_info = _run_cmd( + f'pum info -p pg_qgep -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR}', + error_message='Could not get current version, are you sure the database is accessible ?' + ) + for line in pum_info.splitlines(): + line = line.strip() + if not line: + continue + parts = line.split('|') + if len(parts) > 1: + version = parts[1].strip() + return version + + +def get_available_versions(): + # TODO : this should be done by PUM directly (see https://github.com/opengisch/pum/issues/94) + + if not os.path.exists(DATAMODEL_DELTAS_DIR): + return [] + + versions = set() + for f in os.listdir(DATAMODEL_DELTAS_DIR): + if not os.path.isfile(os.path.join(DATAMODEL_DELTAS_DIR, f)): + continue + if not f.startswith('delta_'): + continue + parts = f.split('_') + versions.add(parts[1]) + return sorted(list(versions)) + + +def upgrade_version(version, srid): + return _run_cmd( + f'pum upgrade -p pg_qgep -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR} -u {version} -v int SRID {srid}', + cwd=os.path.dirname(DATAMODEL_DELTAS_DIR), + error_message='Errors when upgrading the database. Consult logs for more information.' + ) + + +def load_project(): + QgsProject.instance().read(QGIS_PROJECT_PATH) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py new file mode 100644 index 00000000..2860bdc6 --- /dev/null +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -0,0 +1,295 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------- +# +# Profile +# Copyright (C) 2012 Matthias Kuhn +# ----------------------------------------------------------- +# +# licensed under the terms of GNU GPL 2 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, print to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# --------------------------------------------------------------------- + +import os +import configparser +import functools + +from builtins import str +from qgis.PyQt.QtWidgets import QDialog, QMessageBox +from qgis.core import QgsSettings + +from ..utils import get_ui_class +from .. import datamodel_initializer + +DIALOG_UI = get_ui_class('qgepdatamodeldialog.ui') + +QGEP_SERVICE_NAME = 'pg_qgep' + +if os.environ.get('PGSERVICEFILE'): + PG_CONFIG_PATH = os.environ.get('PGSERVICEFILE') +elif os.environ.get('PGSYSCONFDIR'): + PG_CONFIG_PATH = os.path.join(os.environ.get('PGSYSCONFDIR'), 'pg_service.conf') +else: + PG_CONFIG_PATH = ' ~/.pg_service.conf' + + +def qgep_datamodel_error_catcher(func): + """Display QGEPDatamodelError in error messages rather than normal exception dialog""" + + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except datamodel_initializer.QGEPDatamodelError as e: + err = QMessageBox() + err.setText(str(e)) + err.setIcon(QMessageBox.Warning) + err.exec_() + return wrapper + + +class QgepDatamodelInitToolDialog(QDialog, DIALOG_UI): + + def __init__(self, parent=None): + QDialog.__init__(self, parent) + self.setupUi(self) + + self.installDepsButton.pressed.connect(self.install_deps) + + self.pgconfigWriteButton.pressed.connect(self.write_to_pgservice) + self.pgconfigLoadButton.pressed.connect(self.read_from_pgservice) + self.connectionComboBox.currentIndexChanged.connect(self.populate_from_connection_combobox) + + self.pgconfigHostLineEdit.editingFinished.connect(self.update_pgconfig_checks) + self.pgconfigPortLineEdit.editingFinished.connect(self.update_pgconfig_checks) + self.pgconfigDbLineEdit.editingFinished.connect(self.update_pgconfig_checks) + self.pgconfigUserLineEdit.editingFinished.connect(self.update_pgconfig_checks) + self.pgconfigPasswordLineEdit.editingFinished.connect(self.update_pgconfig_checks) + + self.versionCheckButton.pressed.connect(self.update_versions_checks) + self.versionUpgradeButton.pressed.connect(self.upgrade_version) + self.loadProjectButton.pressed.connect(datamodel_initializer.load_project) + + self.checks = { + 'release': False, + 'dependencies': False, + 'pgconfig': False, + 'current_version': False, + } + + @qgep_datamodel_error_catcher + def showEvent(self, event): + self.refresh_connection_combobox() + self.update_requirements_checks() + self.read_from_pgservice() + self.update_versions_checks() + super().showEvent(event) + + def enable_buttons_if_ready(self): + self.versionCheckButton.setEnabled(self.checks['dependencies']) + self.versionUpgradeButton.setEnabled(all(self.checks.values())) + self.loadProjectButton.setEnabled(self.checks['release']) + + @qgep_datamodel_error_catcher + def install_deps(self): + datamodel_initializer.install_deps() + self.update_requirements_checks() + + @qgep_datamodel_error_catcher + def update_requirements_checks(self): + + if datamodel_initializer.check_release_exists(): + self.checks['release'] = True + self.releaseCheckLabel.setText('ok') + self.releaseCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + else: + self.checks['release'] = False + self.releaseCheckLabel.setText('not found') + self.releaseCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + + if datamodel_initializer.check_python_requirements(): + self.checks['dependencies'] = True + self.pythonCheckLabel.setText('ok') + self.pythonCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + else: + self.checks['dependencies'] = False + errors = datamodel_initializer.missing_python_requirements() + self.pythonCheckLabel.setText('\n'.join(f'{dep}: {err}' for dep, err in errors)) + self.pythonCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + + self.enable_buttons_if_ready() + + @qgep_datamodel_error_catcher + def update_pgconfig_checks(self): + + self.checks['pgconfig'] = False + + self.pgconfigCheckLabel.setText('pg_service.conf file missing') + self.pgconfigCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + + if os.path.exists(PG_CONFIG_PATH): + config = configparser.ConfigParser() + config.read(PG_CONFIG_PATH) + + self.pgconfigCheckLabel.setText('pg_qgep configuration missing') + self.pgconfigCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + + if QGEP_SERVICE_NAME in config: + mismatches = [] + if self.pgconfigHostLineEdit.text() != config[QGEP_SERVICE_NAME].get('host'): + mismatches.append('host') + if self.pgconfigPortLineEdit.text() != config[QGEP_SERVICE_NAME].get('port'): + mismatches.append('port') + if self.pgconfigDbLineEdit.text() != config[QGEP_SERVICE_NAME].get('dbname'): + mismatches.append('dbname') + if self.pgconfigUserLineEdit.text() != config[QGEP_SERVICE_NAME].get('user'): + mismatches.append('user') + if self.pgconfigPasswordLineEdit.text() != config[QGEP_SERVICE_NAME].get('password'): + mismatches.append('password') + + if mismatches: + self.pgconfigCheckLabel.setText('mismatches : ' + ','.join(mismatches)) + self.pgconfigCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + else: + self.checks['pgconfig'] = True + self.pgconfigCheckLabel.setText('ok') + self.pgconfigCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + + self.enable_buttons_if_ready() + + @qgep_datamodel_error_catcher + def update_versions_checks(self): + self.checks['current_version'] = False + + current_version = datamodel_initializer.get_current_version() + available_versions = datamodel_initializer.get_available_versions() + + if current_version in available_versions: + self.checks['current_version'] = True + self.versionCheckLabel.setText(current_version) + self.versionCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + elif current_version is None: + self.versionCheckLabel.setText("could not determine version") + self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + else: + self.versionCheckLabel.setText(f"invalid version : {current_version}") + self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + + self.targetVersionComboBox.clear() + for version in reversed(available_versions): + self.targetVersionComboBox.addItem(version) + if version < current_version: + i = self.targetVersionComboBox.model().rowCount() - 1 + self.targetVersionComboBox.model().item(i).setEnabled(False) + + self.enable_buttons_if_ready() + + @qgep_datamodel_error_catcher + def refresh_connection_combobox(self): + self.connectionComboBox.clear() + + self.connectionComboBox.addItem('-- Populate from browser --') + self.connectionComboBox.model().item(0).setEnabled(False) + + settings = QgsSettings() + settings.beginGroup('/PostgreSQL/connections') + for name in settings.childGroups(): + self.connectionComboBox.addItem(name) + settings.endGroup() + + @qgep_datamodel_error_catcher + def read_from_pgservice(self): + + self.pgconfigHostLineEdit.setText('') + self.pgconfigPortLineEdit.setText('') + self.pgconfigDbLineEdit.setText('') + self.pgconfigUserLineEdit.setText('') + self.pgconfigPasswordLineEdit.setText('') + + if os.path.exists(PG_CONFIG_PATH): + config = configparser.ConfigParser() + config.read(PG_CONFIG_PATH) + if QGEP_SERVICE_NAME in config: + self.pgconfigHostLineEdit.setText(config[QGEP_SERVICE_NAME].get('host')) + self.pgconfigPortLineEdit.setText(config[QGEP_SERVICE_NAME].get('port')) + self.pgconfigDbLineEdit.setText(config[QGEP_SERVICE_NAME].get('dbname')) + self.pgconfigUserLineEdit.setText(config[QGEP_SERVICE_NAME].get('user')) + self.pgconfigPasswordLineEdit.setText(config[QGEP_SERVICE_NAME].get('password')) + + self.update_pgconfig_checks() + + @qgep_datamodel_error_catcher + def write_to_pgservice(self): + """ + Saves the selected's postgres to pg_service.conf + """ + + config = configparser.ConfigParser() + if os.path.exists(PG_CONFIG_PATH): + config.read(PG_CONFIG_PATH) + + config[QGEP_SERVICE_NAME] = { + "host": self.pgconfigHostLineEdit.text(), + "port": self.pgconfigPortLineEdit.text(), + "dbname": self.pgconfigDbLineEdit.text(), + "user": self.pgconfigUserLineEdit.text(), + "password": self.pgconfigPasswordLineEdit.text(), + } + + class EqualsSpaceRemover: + # see https://stackoverflow.com/a/25084055/13690651 + output_file = None + + def __init__(self, output_file): + self.output_file = output_file + + def write(self, what): + self.output_file.write(what.replace(" = ", "=", 1)) + + config.write(EqualsSpaceRemover(open(PG_CONFIG_PATH, 'w'))) + + self.update_pgconfig_checks() + + @qgep_datamodel_error_catcher + def populate_from_connection_combobox(self, index): + if index == 0: + return + name = self.connectionComboBox.currentText() + settings = QgsSettings() + settings.beginGroup(f"/PostgreSQL/connections/{name}") + self.pgconfigHostLineEdit.setText(settings.value("host", "", type=str)) + self.pgconfigPortLineEdit.setText(settings.value("port", "", type=str)) + self.pgconfigDbLineEdit.setText(settings.value("database", "", type=str)) + self.pgconfigUserLineEdit.setText(settings.value("username", "", type=str)) + self.pgconfigPasswordLineEdit.setText(settings.value("password", "", type=str)) + settings.endGroup() + self.update_pgconfig_checks() + + @qgep_datamodel_error_catcher + def upgrade_version(self): + version = self.targetVersionComboBox.currentText() + + confirm = QMessageBox() + confirm.setText(f"You are about to update your datamodel to version {version}. ") + confirm.setInformativeText( + "Please confirm that you have a backup of your data as this operation can result in data loss." + ) + confirm.setStandardButtons(QMessageBox.Apply | QMessageBox.Cancel) + confirm.setIcon(QMessageBox.Warning) + + if confirm.exec_() == QMessageBox.Apply: + datamodel_initializer.upgrade_version(version, self.sridLineEdit.text()) + self.update_versions_checks() diff --git a/qgepplugin/qgepplugin.py b/qgepplugin/qgepplugin.py index c961b01c..7603f19b 100644 --- a/qgepplugin/qgepplugin.py +++ b/qgepplugin/qgepplugin.py @@ -47,6 +47,7 @@ from .gui.qgepprofiledockwidget import QgepProfileDockWidget from .gui.qgepplotsvgwidget import QgepPlotSVGWidget from .gui.qgepsettingsdialog import QgepSettingsDialog +from .gui.qgepdatamodeldialog import QgepDatamodelInitToolDialog from .gui.qgepwizard import QgepWizard from .utils.qgeplogging import QgepQgsLogHandler from .utils.translation import setup_i18n @@ -209,6 +210,10 @@ def initGui(self): self.tr('Settings'), self.iface.mainWindow()) self.settingsAction.triggered.connect(self.showSettings) + self.datamodelInitToolAction = QAction( + self.tr('Datamodel tool'), self.iface.mainWindow()) + self.datamodelInitToolAction.triggered.connect(self.showDatamodelInitTool) + # Add toolbar button and menu item self.toolbar = QToolBar(QApplication.translate('qgepplugin', 'QGEP')) self.toolbar.addAction(self.profileAction) @@ -219,6 +224,7 @@ def initGui(self): self.toolbar.addAction(self.connectNetworkElementsAction) self.iface.addPluginToMenu("&QGEP", self.profileAction) + self.iface.addPluginToMenu("&QGEP", self.datamodelInitToolAction) self.iface.addPluginToMenu("&QGEP", self.settingsAction) self.iface.addPluginToMenu("&QGEP", self.aboutAction) @@ -388,3 +394,8 @@ def about(self): def showSettings(self): settings_dlg = QgepSettingsDialog(self.iface.mainWindow()) settings_dlg.exec_() + + def showDatamodelInitTool(self): + if not hasattr(self, '_datamodel_dlg'): + self.datamodel_dlg = QgepDatamodelInitToolDialog(self.iface.mainWindow()) + self.datamodel_dlg.show() \ No newline at end of file diff --git a/qgepplugin/ui/qgepdatamodeldialog.ui b/qgepplugin/ui/qgepdatamodeldialog.ui new file mode 100644 index 00000000..00a8e155 --- /dev/null +++ b/qgepplugin/ui/qgepdatamodeldialog.ui @@ -0,0 +1,355 @@ + + + QgepSettingsDialog + + + + 0 + 0 + 500 + 478 + + + + Datamodel tool + + + + + + Depdencies + + + + + + Release files + + + + + + + color: rgb(170, 0, 0); +font-weight: bold; + + + Unknown + + + Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse + + + + + + + Python requirements + + + + + + + color: rgb(170, 0, 0); +font-weight: bold; + + + Unknown + + + Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse + + + + + + + Install dependencies + + + + + + + + + + Postgres service configuration + + + + + + PG Config setup + + + + + + + color: rgb(170, 0, 0); +font-weight: bold; + + + Unknown + + + + + + + Host + + + + + + + + + + Port + + + + + + + + + + Database + + + + + + + + + + User + + + + + + + + + + Password + + + + + + + + + + + + Write to pg_service.conf + + + + + + + Reload from pg_service.conf + + + + + + + + + + + + + + + Datamodel + + + + + + Current version + + + + + + + + + color: rgb(170, 0, 0); +font-weight: bold; + + + Unknown + + + + + + + Refresh + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + Target version + + + + + + + + + + + + Upgrade + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + SRID + + + + + + + 2056 + + + + + + + + + + QGIS project + + + + + + Load QGIS project template + + + + + + + Load + + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Close + + + + + + + + + mBtnBoxOkCancel + accepted() + QgepSettingsDialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + mBtnBoxOkCancel + rejected() + QgepSettingsDialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + From 1ff74e1c06f449e380c2c8d7e4d6c28563f2f08a Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 23 Oct 2020 16:45:03 +0200 Subject: [PATCH 02/35] [feature] improve init tool --- qgepplugin/datamodel_initializer/__init__.py | 69 ++++- qgepplugin/gui/qgepdatamodeldialog.py | 251 ++++++++----------- qgepplugin/ui/qgepdatamodeldialog.ui | 97 ++----- qgepplugin/ui/qgeppgserviceeditordialog.ui | 152 +++++++++++ 4 files changed, 340 insertions(+), 229 deletions(-) create mode 100644 qgepplugin/ui/qgeppgserviceeditordialog.ui diff --git a/qgepplugin/datamodel_initializer/__init__.py b/qgepplugin/datamodel_initializer/__init__.py index 61439c43..744e1a43 100644 --- a/qgepplugin/datamodel_initializer/__init__.py +++ b/qgepplugin/datamodel_initializer/__init__.py @@ -6,9 +6,11 @@ import pkg_resources import site import importlib +import configparser from qgis.PyQt.QtCore import QUrl, QFile, QIODevice from qgis.PyQt.QtNetwork import QNetworkRequest +from qgis.PyQt.QtXml import QDomDocument from qgis.core import QgsNetworkAccessManager, QgsFeedback, QgsMessageLog, Qgis, QgsProject @@ -16,6 +18,14 @@ DATAMODEL_RELEASE = '1.5.2' QGEP_RELEASE = '8.0' +# Path for pg_service.conf +if os.environ.get('PGSERVICEFILE'): + PG_CONFIG_PATH = os.environ.get('PGSERVICEFILE') +elif os.environ.get('PGSYSCONFDIR'): + PG_CONFIG_PATH = os.path.join(os.environ.get('PGSYSCONFDIR'), 'pg_service.conf') +else: + PG_CONFIG_PATH = ' ~/.pg_service.conf' + # Derived urls/paths, may require adaptations if release structure changes DATAMODEL_RELEASE_URL = f'https://github.com/QGEP/datamodel/archive/v{DATAMODEL_RELEASE}.zip' QGEP_RELEASE_URL = f'https://github.com/QGEP/QGEP/releases/download/v{QGEP_RELEASE}/qgep.zip' @@ -131,12 +141,12 @@ def install_deps(): importlib.reload(site) -def get_current_version(): +def get_current_version(pgservice): if not os.path.exists(DATAMODEL_DELTAS_DIR): return None pum_info = _run_cmd( - f'pum info -p pg_qgep -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR}', + f'pum info -p {pgservice} -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR}', error_message='Could not get current version, are you sure the database is accessible ?' ) for line in pum_info.splitlines(): @@ -166,13 +176,60 @@ def get_available_versions(): return sorted(list(versions)) -def upgrade_version(version, srid): +def upgrade_version(pgservice, version, srid): + try: + get_current_version(pgservice) + except QGEPDatamodelError: + # TODO : this should be done by PUM directly (see https://github.com/opengisch/pum/issues/94) + return _run_cmd( + f'pum baseline -p {pgservice} -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR} -b 0.0.0', + cwd=os.path.dirname(DATAMODEL_DELTAS_DIR), + error_message='Errors when initializing the database. Consult logs for more information.' + ) return _run_cmd( - f'pum upgrade -p pg_qgep -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR} -u {version} -v int SRID {srid}', + f'pum upgrade -p {pgservice} -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR} -u {version} -v int SRID {srid}', cwd=os.path.dirname(DATAMODEL_DELTAS_DIR), error_message='Errors when upgrading the database. Consult logs for more information.' ) -def load_project(): - QgsProject.instance().read(QGIS_PROJECT_PATH) +def load_project(pgservice): + with open(QGIS_PROJECT_PATH, 'r') as original_project: + contents = original_project.read() + + # replace the service name + contents = contents.replace("service='pg_qgep'", f"service='{pgservice}'") + + output_file = tempfile.NamedTemporaryFile(suffix='.qgs', delete=False) + output_file.write(contents.encode('utf8')) + + QgsProject.instance().read(output_file.name) + + +def read_pgservice(): + config = configparser.ConfigParser() + if os.path.exists(PG_CONFIG_PATH): + config.read(PG_CONFIG_PATH) + return config + + +def get_pgservice_configs_names(): + config = read_pgservice() + return config.sections() + + +def write_pgservice_conf(service_name, config_dict): + config = read_pgservice() + config[service_name] = config_dict + + class EqualsSpaceRemover: + # see https://stackoverflow.com/a/25084055/13690651 + output_file = None + + def __init__(self, output_file): + self.output_file = output_file + + def write(self, what): + self.output_file.write(what.replace(" = ", "=", 1)) + + config.write(EqualsSpaceRemover(open(PG_CONFIG_PATH, 'w'))) \ No newline at end of file diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 2860bdc6..02d33b7a 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -29,22 +29,11 @@ from builtins import str from qgis.PyQt.QtWidgets import QDialog, QMessageBox -from qgis.core import QgsSettings +from qgis.core import QgsSettings, QgsMessageLog from ..utils import get_ui_class from .. import datamodel_initializer -DIALOG_UI = get_ui_class('qgepdatamodeldialog.ui') - -QGEP_SERVICE_NAME = 'pg_qgep' - -if os.environ.get('PGSERVICEFILE'): - PG_CONFIG_PATH = os.environ.get('PGSERVICEFILE') -elif os.environ.get('PGSYSCONFDIR'): - PG_CONFIG_PATH = os.path.join(os.environ.get('PGSYSCONFDIR'), 'pg_service.conf') -else: - PG_CONFIG_PATH = ' ~/.pg_service.conf' - def qgep_datamodel_error_catcher(func): """Display QGEPDatamodelError in error messages rather than normal exception dialog""" @@ -61,7 +50,38 @@ def wrapper(*args, **kwargs): return wrapper -class QgepDatamodelInitToolDialog(QDialog, DIALOG_UI): + +class QgepPgserviceEditorDialog(QDialog, get_ui_class('qgeppgserviceeditordialog.ui')): + + def __init__(self, existing_config_names, parent=None): + QDialog.__init__(self, parent) + self.setupUi(self) + self.existing_config_names = existing_config_names + self.nameLineEdit.textChanged.connect(self.check_name) + + def check_name(self, new_text): + if new_text in self.existing_config_names: + self.nameCheckLabel.setText('will overwrite') + self.nameCheckLabel.setStyleSheet('color: rgb(170, 65, 0);\nfont-weight: bold;') + else: + self.nameCheckLabel.setText('will be created') + self.nameCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + + def conf_name(self): + return self.nameLineEdit.text() + + def conf_dict(self): + return { + "host": self.pgconfigHostLineEdit.text(), + "port": self.pgconfigPortLineEdit.text(), + "dbname": self.pgconfigDbLineEdit.text(), + "user": self.pgconfigUserLineEdit.text(), + "password": self.pgconfigPasswordLineEdit.text(), + } + + + +class QgepDatamodelInitToolDialog(QDialog, get_ui_class('qgepdatamodeldialog.ui')): def __init__(self, parent=None): QDialog.__init__(self, parent) @@ -69,19 +89,11 @@ def __init__(self, parent=None): self.installDepsButton.pressed.connect(self.install_deps) - self.pgconfigWriteButton.pressed.connect(self.write_to_pgservice) - self.pgconfigLoadButton.pressed.connect(self.read_from_pgservice) - self.connectionComboBox.currentIndexChanged.connect(self.populate_from_connection_combobox) + self.pgserviceComboBox.activated.connect(self.update_pgconfig_checks) + self.pgserviceAddButton.pressed.connect(self.add_pgconfig) - self.pgconfigHostLineEdit.editingFinished.connect(self.update_pgconfig_checks) - self.pgconfigPortLineEdit.editingFinished.connect(self.update_pgconfig_checks) - self.pgconfigDbLineEdit.editingFinished.connect(self.update_pgconfig_checks) - self.pgconfigUserLineEdit.editingFinished.connect(self.update_pgconfig_checks) - self.pgconfigPasswordLineEdit.editingFinished.connect(self.update_pgconfig_checks) - - self.versionCheckButton.pressed.connect(self.update_versions_checks) self.versionUpgradeButton.pressed.connect(self.upgrade_version) - self.loadProjectButton.pressed.connect(datamodel_initializer.load_project) + self.loadProjectButton.pressed.connect(self.load_project) self.checks = { 'release': False, @@ -92,14 +104,14 @@ def __init__(self, parent=None): @qgep_datamodel_error_catcher def showEvent(self, event): - self.refresh_connection_combobox() + self.refresh_pgservice_combobox() self.update_requirements_checks() - self.read_from_pgservice() + self.update_pgconfig_checks() self.update_versions_checks() + self.pgservicePathLabel.setText(datamodel_initializer.PG_CONFIG_PATH) super().showEvent(event) def enable_buttons_if_ready(self): - self.versionCheckButton.setEnabled(self.checks['dependencies']) self.versionUpgradeButton.setEnabled(all(self.checks.values())) self.loadProjectButton.setEnabled(self.checks['release']) @@ -133,53 +145,65 @@ def update_requirements_checks(self): self.enable_buttons_if_ready() @qgep_datamodel_error_catcher - def update_pgconfig_checks(self): + def refresh_pgservice_combobox(self): + self.pgserviceComboBox.clear() + config_names = datamodel_initializer.get_pgservice_configs_names() + for config_name in config_names: + self.pgserviceComboBox.addItem(config_name) + self.pgserviceComboBox.setCurrentIndex(-1) - self.checks['pgconfig'] = False - - self.pgconfigCheckLabel.setText('pg_service.conf file missing') - self.pgconfigCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + @qgep_datamodel_error_catcher + def add_pgconfig(self): + existing_config_names = datamodel_initializer.get_pgservice_configs_names() + add_dialog = QgepPgserviceEditorDialog(existing_config_names) + if add_dialog.exec_() == QDialog.Accepted: + name = add_dialog.conf_name() + conf = add_dialog.conf_dict() + datamodel_initializer.write_pgservice_conf(name, conf) + self.refresh_pgservice_combobox() + self.pgserviceComboBox.setCurrentIndex(self.pgserviceComboBox.findText(name)) + self.update_pgconfig_checks() - if os.path.exists(PG_CONFIG_PATH): - config = configparser.ConfigParser() - config.read(PG_CONFIG_PATH) + @qgep_datamodel_error_catcher + def update_pgconfig_checks(self, _=None): - self.pgconfigCheckLabel.setText('pg_qgep configuration missing') + if self.pgserviceComboBox.currentText(): + self.checks['pgconfig'] = True + self.pgconfigCheckLabel.setText('ok') + self.pgconfigCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + else: + self.checks['pgconfig'] = False + self.pgconfigCheckLabel.setText('not set') self.pgconfigCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') - if QGEP_SERVICE_NAME in config: - mismatches = [] - if self.pgconfigHostLineEdit.text() != config[QGEP_SERVICE_NAME].get('host'): - mismatches.append('host') - if self.pgconfigPortLineEdit.text() != config[QGEP_SERVICE_NAME].get('port'): - mismatches.append('port') - if self.pgconfigDbLineEdit.text() != config[QGEP_SERVICE_NAME].get('dbname'): - mismatches.append('dbname') - if self.pgconfigUserLineEdit.text() != config[QGEP_SERVICE_NAME].get('user'): - mismatches.append('user') - if self.pgconfigPasswordLineEdit.text() != config[QGEP_SERVICE_NAME].get('password'): - mismatches.append('password') - - if mismatches: - self.pgconfigCheckLabel.setText('mismatches : ' + ','.join(mismatches)) - self.pgconfigCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') - else: - self.checks['pgconfig'] = True - self.pgconfigCheckLabel.setText('ok') - self.pgconfigCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') - - self.enable_buttons_if_ready() + self.update_versions_checks() @qgep_datamodel_error_catcher def update_versions_checks(self): self.checks['current_version'] = False + + available_versions = datamodel_initializer.get_available_versions() + self.targetVersionComboBox.clear() + for version in reversed(available_versions): + self.targetVersionComboBox.addItem(version) - current_version = datamodel_initializer.get_current_version() - available_versions = datamodel_initializer.get_available_versions() + pgservice = self.pgserviceComboBox.currentText() + if not pgservice: + self.versionCheckLabel.setText('service not selected') + self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + return - if current_version in available_versions: + try: + current_version = datamodel_initializer.get_current_version(pgservice) + except datamodel_initializer.QGEPDatamodelError: + # Can happend if PUM is not initialized, unfortunately we can't really + # determine if this is a connection error or if PUM is not initailized + # see https://github.com/opengisch/pum/issues/96 + current_version = None + + if current_version is None or current_version in available_versions: self.checks['current_version'] = True - self.versionCheckLabel.setText(current_version) + self.versionCheckLabel.setText(current_version or 'not initialized') self.versionCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') elif current_version is None: self.versionCheckLabel.setText("could not determine version") @@ -188,102 +212,22 @@ def update_versions_checks(self): self.versionCheckLabel.setText(f"invalid version : {current_version}") self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') - self.targetVersionComboBox.clear() - for version in reversed(available_versions): - self.targetVersionComboBox.addItem(version) - if version < current_version: - i = self.targetVersionComboBox.model().rowCount() - 1 - self.targetVersionComboBox.model().item(i).setEnabled(False) + # disable unapplicable versions + for i in range(self.targetVersionComboBox.model().rowCount()): + item_version = self.targetVersionComboBox.model().item(i).text() + QgsMessageLog.logMessage(item_version) + enabled = current_version is None or item_version >= current_version + self.targetVersionComboBox.model().item(i).setEnabled(enabled) self.enable_buttons_if_ready() - @qgep_datamodel_error_catcher - def refresh_connection_combobox(self): - self.connectionComboBox.clear() - - self.connectionComboBox.addItem('-- Populate from browser --') - self.connectionComboBox.model().item(0).setEnabled(False) - - settings = QgsSettings() - settings.beginGroup('/PostgreSQL/connections') - for name in settings.childGroups(): - self.connectionComboBox.addItem(name) - settings.endGroup() - - @qgep_datamodel_error_catcher - def read_from_pgservice(self): - - self.pgconfigHostLineEdit.setText('') - self.pgconfigPortLineEdit.setText('') - self.pgconfigDbLineEdit.setText('') - self.pgconfigUserLineEdit.setText('') - self.pgconfigPasswordLineEdit.setText('') - - if os.path.exists(PG_CONFIG_PATH): - config = configparser.ConfigParser() - config.read(PG_CONFIG_PATH) - if QGEP_SERVICE_NAME in config: - self.pgconfigHostLineEdit.setText(config[QGEP_SERVICE_NAME].get('host')) - self.pgconfigPortLineEdit.setText(config[QGEP_SERVICE_NAME].get('port')) - self.pgconfigDbLineEdit.setText(config[QGEP_SERVICE_NAME].get('dbname')) - self.pgconfigUserLineEdit.setText(config[QGEP_SERVICE_NAME].get('user')) - self.pgconfigPasswordLineEdit.setText(config[QGEP_SERVICE_NAME].get('password')) - - self.update_pgconfig_checks() - - @qgep_datamodel_error_catcher - def write_to_pgservice(self): - """ - Saves the selected's postgres to pg_service.conf - """ - - config = configparser.ConfigParser() - if os.path.exists(PG_CONFIG_PATH): - config.read(PG_CONFIG_PATH) - - config[QGEP_SERVICE_NAME] = { - "host": self.pgconfigHostLineEdit.text(), - "port": self.pgconfigPortLineEdit.text(), - "dbname": self.pgconfigDbLineEdit.text(), - "user": self.pgconfigUserLineEdit.text(), - "password": self.pgconfigPasswordLineEdit.text(), - } - - class EqualsSpaceRemover: - # see https://stackoverflow.com/a/25084055/13690651 - output_file = None - - def __init__(self, output_file): - self.output_file = output_file - - def write(self, what): - self.output_file.write(what.replace(" = ", "=", 1)) - - config.write(EqualsSpaceRemover(open(PG_CONFIG_PATH, 'w'))) - - self.update_pgconfig_checks() - - @qgep_datamodel_error_catcher - def populate_from_connection_combobox(self, index): - if index == 0: - return - name = self.connectionComboBox.currentText() - settings = QgsSettings() - settings.beginGroup(f"/PostgreSQL/connections/{name}") - self.pgconfigHostLineEdit.setText(settings.value("host", "", type=str)) - self.pgconfigPortLineEdit.setText(settings.value("port", "", type=str)) - self.pgconfigDbLineEdit.setText(settings.value("database", "", type=str)) - self.pgconfigUserLineEdit.setText(settings.value("username", "", type=str)) - self.pgconfigPasswordLineEdit.setText(settings.value("password", "", type=str)) - settings.endGroup() - self.update_pgconfig_checks() - @qgep_datamodel_error_catcher def upgrade_version(self): version = self.targetVersionComboBox.currentText() + pgservice = self.pgserviceComboBox.currentText() confirm = QMessageBox() - confirm.setText(f"You are about to update your datamodel to version {version}. ") + confirm.setText(f"You are about to update the datamodel on {pgservice} to version {version}. ") confirm.setInformativeText( "Please confirm that you have a backup of your data as this operation can result in data loss." ) @@ -291,5 +235,10 @@ def upgrade_version(self): confirm.setIcon(QMessageBox.Warning) if confirm.exec_() == QMessageBox.Apply: - datamodel_initializer.upgrade_version(version, self.sridLineEdit.text()) + datamodel_initializer.upgrade_version(pgservice, version, self.sridLineEdit.text()) self.update_versions_checks() + + @qgep_datamodel_error_catcher + def load_project(self): + pgservice = self.pgserviceComboBox.currentText() + datamodel_initializer.load_project(pgservice) \ No newline at end of file diff --git a/qgepplugin/ui/qgepdatamodeldialog.ui b/qgepplugin/ui/qgepdatamodeldialog.ui index 00a8e155..ed4f2a7e 100644 --- a/qgepplugin/ui/qgepdatamodeldialog.ui +++ b/qgepplugin/ui/qgepdatamodeldialog.ui @@ -78,95 +78,55 @@ font-weight: bold; Postgres service configuration - - - - PG Config setup - - - - - - - color: rgb(170, 0, 0); -font-weight: bold; - - - Unknown - - - - - - - Host - - - - - - - + - Port + File location - - - - - - Database - - - - - - - - - - User + + + font: italic; color: #888 - - - - - - - - Password + TextLabel - - - - + - + + + + - Write to pg_service.conf + + - + + + color: rgb(170, 0, 0); +font-weight: bold; + - Reload from pg_service.conf + Unknown - - - + + + + PG Config setup + + + @@ -196,13 +156,6 @@ font-weight: bold; - - - - Refresh - - - diff --git a/qgepplugin/ui/qgeppgserviceeditordialog.ui b/qgepplugin/ui/qgeppgserviceeditordialog.ui new file mode 100644 index 00000000..b235d574 --- /dev/null +++ b/qgepplugin/ui/qgeppgserviceeditordialog.ui @@ -0,0 +1,152 @@ + + + Dialog + + + + 0 + 0 + 435 + 204 + + + + PG Config editor + + + + + + Config name + + + + + + + + + + + + color: rgb(170, 0, 0); +font-weight: bold; + + + Unknown + + + + + + + + + Host + + + + + + + + + + Port + + + + + + + + + + Database + + + + + + + + + + User + + + + + + + + + + Password + + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Save + + + + + + + + + buttonBox + accepted() + Dialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + Dialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + From 5997357fbef44e9e3ee3acb5e079a4882b8e5650 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Wed, 28 Oct 2020 11:19:32 +0100 Subject: [PATCH 03/35] avancement --- qgepplugin/datamodel_initializer/__init__.py | 63 +++++++++++++------- qgepplugin/gui/qgepdatamodeldialog.py | 11 ++-- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/qgepplugin/datamodel_initializer/__init__.py b/qgepplugin/datamodel_initializer/__init__.py index 744e1a43..6711ca7e 100644 --- a/qgepplugin/datamodel_initializer/__init__.py +++ b/qgepplugin/datamodel_initializer/__init__.py @@ -7,6 +7,7 @@ import site import importlib import configparser +import psycopg2 from qgis.PyQt.QtCore import QUrl, QFile, QIODevice from qgis.PyQt.QtNetwork import QNetworkRequest @@ -56,6 +57,18 @@ def _run_cmd(shell_command, cwd=None, error_message='Subprocess error, see logs return result.stdout.decode() +def _download(url, filename, feedback): + network_manager = QgsNetworkAccessManager.instance() + QgsMessageLog.logMessage(f"Downloading {url} to {RELEASE_DIR}", "QGEP") + reply = network_manager.blockingGet(QNetworkRequest(QUrl(DATAMODEL_RELEASE_URL)), feedback=feedback) + download_path = os.path.join(RELEASE_DIR, filename) + download_file = QFile(download_path) + download_file.open(QIODevice.WriteOnly) + download_file.write(reply.content()) + download_file.close() + return download_file.fileName() + + def check_release_exists(): return os.path.exists(QGIS_PROJECT_PATH) and os.path.exists(DATAMODEL_DIR) @@ -84,7 +97,6 @@ def missing_python_requirements(): def install_deps(): - network_manager = QgsNetworkAccessManager.instance() feedback = QgsFeedback() # Download the files if needed @@ -95,30 +107,18 @@ def install_deps(): else: os.makedirs(RELEASE_DIR, exist_ok=True) - # Download datamodel - QgsMessageLog.logMessage(f"Downloading {DATAMODEL_RELEASE_URL} to {RELEASE_DIR}", "QGEP") - reply = network_manager.blockingGet(QNetworkRequest(QUrl(DATAMODEL_RELEASE_URL)), feedback=feedback) - datamodel_file = QFile(os.path.join(RELEASE_DIR, 'datamodel.zip')) - datamodel_file.open(QIODevice.WriteOnly) - datamodel_file.write(reply.content()) - datamodel_file.close() - - # Download QGEP - QgsMessageLog.logMessage(f"Downloading {QGEP_RELEASE_URL} to {RELEASE_DIR}", "QGEP") - reply = network_manager.blockingGet(QNetworkRequest(QUrl(QGEP_RELEASE_URL)), feedback=feedback) - qgep_file = QFile(os.path.join(RELEASE_DIR, 'qgep.zip')) - qgep_file.open(QIODevice.WriteOnly) - qgep_file.write(reply.content()) - qgep_file.close() + # Download files + datamodel_path = _download(DATAMODEL_RELEASE_URL, 'datamodel.zip', feedback=feedback) + qgep_path = _download(QGEP_RELEASE_URL, 'qgep.zip', feedback=feedback) QgsMessageLog.logMessage(f"Extracting files to {RELEASE_DIR}", "QGEP") # Unzip datamodel - datamodel_zip = zipfile.ZipFile(datamodel_file.fileName()) + datamodel_zip = zipfile.ZipFile(datamodel_path) datamodel_zip.extractall(RELEASE_DIR) # Unzip QGEP - qgep_zip = zipfile.ZipFile(qgep_file.fileName()) + qgep_zip = zipfile.ZipFile(qgep_path) qgep_zip.extractall(RELEASE_DIR) # TODO : Ideally, this should be done in a venv, as to avoid permission issues and/or modification @@ -178,14 +178,37 @@ def get_available_versions(): def upgrade_version(pgservice, version, srid): try: + # If we can't get current version, it's probably that the DB is not initialized + # (or maybe we can't connect, but we can't know easily with PUM) get_current_version(pgservice) except QGEPDatamodelError: + + QgsMessageLog.logMessage("Upgrading failed, trying to initialize the datamodel", "QGEP") + + feedback = QgsFeedback() + # TODO : this should be done by PUM directly (see https://github.com/opengisch/pum/issues/94) - return _run_cmd( + # also currently SRID doesn't work + try: + sql_path = _download(DATAMODEL_STRUCTURE_URL.format(version=version), f"structure_with_value_lists-{version}.sql", feedback) + conn = psycopg2.connect(f"service={pgservice}") + cur = conn.cursor() + cur.execute('CREATE SCHEMA IF NOT EXISTS qgep_sys;') + cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;') + cur.execute(open(sql_path, "r").read()) + conn.commit() + cur.close() + conn.close() + except psycopg2.Error as e: + raise QGEPDatamodelError(str(e)) + + # TODO : this also should be done by pum upgrade directly (see https://github.com/opengisch/pum/issues/94) + _run_cmd( f'pum baseline -p {pgservice} -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR} -b 0.0.0', cwd=os.path.dirname(DATAMODEL_DELTAS_DIR), error_message='Errors when initializing the database. Consult logs for more information.' ) + return _run_cmd( f'pum upgrade -p {pgservice} -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR} -u {version} -v int SRID {srid}', cwd=os.path.dirname(DATAMODEL_DELTAS_DIR), @@ -221,7 +244,7 @@ def get_pgservice_configs_names(): def write_pgservice_conf(service_name, config_dict): config = read_pgservice() config[service_name] = config_dict - + class EqualsSpaceRemover: # see https://stackoverflow.com/a/25084055/13690651 output_file = None diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 02d33b7a..517c7cae 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -181,8 +181,8 @@ def update_pgconfig_checks(self, _=None): @qgep_datamodel_error_catcher def update_versions_checks(self): self.checks['current_version'] = False - - available_versions = datamodel_initializer.get_available_versions() + + available_versions = datamodel_initializer.get_available_versions() self.targetVersionComboBox.clear() for version in reversed(available_versions): self.targetVersionComboBox.addItem(version) @@ -201,7 +201,7 @@ def update_versions_checks(self): # see https://github.com/opengisch/pum/issues/96 current_version = None - if current_version is None or current_version in available_versions: + if current_version is None or current_version == '0.0.0' or current_version in available_versions: self.checks['current_version'] = True self.versionCheckLabel.setText(current_version or 'not initialized') self.versionCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') @@ -215,7 +215,6 @@ def update_versions_checks(self): # disable unapplicable versions for i in range(self.targetVersionComboBox.model().rowCount()): item_version = self.targetVersionComboBox.model().item(i).text() - QgsMessageLog.logMessage(item_version) enabled = current_version is None or item_version >= current_version self.targetVersionComboBox.model().item(i).setEnabled(enabled) @@ -239,6 +238,6 @@ def upgrade_version(self): self.update_versions_checks() @qgep_datamodel_error_catcher - def load_project(self): + def load_project(self): pgservice = self.pgserviceComboBox.currentText() - datamodel_initializer.load_project(pgservice) \ No newline at end of file + datamodel_initializer.load_project(pgservice) From 643cfdd373d2bb52b87742647fbf53dcfd1415ea Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Wed, 28 Oct 2020 12:55:19 +0100 Subject: [PATCH 04/35] avancement --- qgepplugin/datamodel_initializer/__init__.py | 139 +++++++++++-------- qgepplugin/gui/qgepdatamodeldialog.py | 39 +++++- 2 files changed, 114 insertions(+), 64 deletions(-) diff --git a/qgepplugin/datamodel_initializer/__init__.py b/qgepplugin/datamodel_initializer/__init__.py index 6711ca7e..84dce48e 100644 --- a/qgepplugin/datamodel_initializer/__init__.py +++ b/qgepplugin/datamodel_initializer/__init__.py @@ -11,13 +11,13 @@ from qgis.PyQt.QtCore import QUrl, QFile, QIODevice from qgis.PyQt.QtNetwork import QNetworkRequest -from qgis.PyQt.QtXml import QDomDocument -from qgis.core import QgsNetworkAccessManager, QgsFeedback, QgsMessageLog, Qgis, QgsProject +from qgis.core import QgsNetworkAccessManager, QgsMessageLog, Qgis, QgsProject # Basic config -DATAMODEL_RELEASE = '1.5.2' +MAIN_DATAMODEL_RELEASE = '1.5.2' QGEP_RELEASE = '8.0' +TEMP_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', 'datamodel-init') # Path for pg_service.conf if os.environ.get('PGSERVICEFILE'): @@ -28,14 +28,12 @@ PG_CONFIG_PATH = ' ~/.pg_service.conf' # Derived urls/paths, may require adaptations if release structure changes -DATAMODEL_RELEASE_URL = f'https://github.com/QGEP/datamodel/archive/v{DATAMODEL_RELEASE}.zip' -QGEP_RELEASE_URL = f'https://github.com/QGEP/QGEP/releases/download/v{QGEP_RELEASE}/qgep.zip' -hash = hashlib.md5((DATAMODEL_RELEASE_URL + QGEP_RELEASE_URL).encode('utf-8')).hexdigest()[0:8] -RELEASE_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', hash) -QGIS_PROJECT_PATH = os.path.join(RELEASE_DIR, 'project', 'qgep.qgs') -DATAMODEL_DIR = os.path.join(RELEASE_DIR, f'datamodel-{DATAMODEL_RELEASE}') -REQUIREMENTS_PATH = os.path.join(DATAMODEL_DIR, 'requirements.txt') -DATAMODEL_DELTAS_DIR = os.path.join(DATAMODEL_DIR, "delta") +MAIN_DATAMODEL_RELEASE_URL = f'https://github.com/QGEP/datamodel/archive/{MAIN_DATAMODEL_RELEASE}.zip' +QGEP_RELEASE_URL = f'https://github.com/QGEP/QGEP/releases/download/v{QGEP_RELEASE}/qgep.zip' # TODO : we should download the release according the the current datamodel version, not the latest one +REQUIREMENTS_PATH = os.path.join(TEMP_DIR, f'datamodel-{MAIN_DATAMODEL_RELEASE}', 'requirements.txt') +DATAMODEL_DELTAS_DIR = os.path.join(TEMP_DIR, f'datamodel-{MAIN_DATAMODEL_RELEASE}', "delta") +QGIS_PROJECT_PATH = os.path.join(TEMP_DIR, 'project', 'qgep.qgs') +INITIAL_STRUCTURE_URL_TEMPLATE = "https://github.com/QGEP/datamodel/releases/download/{version}/qgep_v{version}_structure_with_value_lists.sql" class QGEPDatamodelError(Exception): @@ -57,31 +55,30 @@ def _run_cmd(shell_command, cwd=None, error_message='Subprocess error, see logs return result.stdout.decode() -def _download(url, filename, feedback): +def _download(url, filename): + os.makedirs(TEMP_DIR, exist_ok=True) + network_manager = QgsNetworkAccessManager.instance() - QgsMessageLog.logMessage(f"Downloading {url} to {RELEASE_DIR}", "QGEP") - reply = network_manager.blockingGet(QNetworkRequest(QUrl(DATAMODEL_RELEASE_URL)), feedback=feedback) - download_path = os.path.join(RELEASE_DIR, filename) + QgsMessageLog.logMessage(f"Downloading {url} to {TEMP_DIR}", "QGEP") + reply = network_manager.blockingGet(QNetworkRequest(QUrl(url))) + download_path = os.path.join(TEMP_DIR, filename) download_file = QFile(download_path) download_file.open(QIODevice.WriteOnly) download_file.write(reply.content()) download_file.close() return download_file.fileName() - def check_release_exists(): - return os.path.exists(QGIS_PROJECT_PATH) and os.path.exists(DATAMODEL_DIR) - + return os.path.exists(REQUIREMENTS_PATH) and os.path.exists(DATAMODEL_DELTAS_DIR) def check_python_requirements(): return len(missing_python_requirements()) == 0 - def missing_python_requirements(): # see https://stackoverflow.com/a/45474387/13690651 missing = [] - if not os.path.exists(REQUIREMENTS_PATH): + if not check_release_exists(): missing.append(('unknown', 'requirements not found')) else: requirements = pkg_resources.parse_requirements(open(REQUIREMENTS_PATH)) @@ -94,32 +91,20 @@ def missing_python_requirements(): missing.append((requirement, 'conflict')) return missing - -def install_deps(): - - feedback = QgsFeedback() +def install_deps(progress_dialog): # Download the files if needed - if check_release_exists(): - QgsMessageLog.logMessage(f"Required files are already present in {RELEASE_DIR}", "QGEP") - - else: - os.makedirs(RELEASE_DIR, exist_ok=True) + progress_dialog.set_action("Downloading the release") + if not check_release_exists(): # Download files - datamodel_path = _download(DATAMODEL_RELEASE_URL, 'datamodel.zip', feedback=feedback) - qgep_path = _download(QGEP_RELEASE_URL, 'qgep.zip', feedback=feedback) - - QgsMessageLog.logMessage(f"Extracting files to {RELEASE_DIR}", "QGEP") + datamodel_path = _download(MAIN_DATAMODEL_RELEASE_URL, 'datamodel.zip') - # Unzip datamodel + # Unzip + QgsMessageLog.logMessage(f"Extracting files to {TEMP_DIR}", "QGEP") datamodel_zip = zipfile.ZipFile(datamodel_path) - datamodel_zip.extractall(RELEASE_DIR) - - # Unzip QGEP - qgep_zip = zipfile.ZipFile(qgep_path) - qgep_zip.extractall(RELEASE_DIR) + datamodel_zip.extractall(TEMP_DIR) # TODO : Ideally, this should be done in a venv, as to avoid permission issues and/or modification # of libraries versions that could affect other parts of the system. @@ -133,13 +118,12 @@ def install_deps(): # Anyway, pip deps support should be done in QGIS one day so all plugins can benefit. # In the mean time we just install globally and hope for the best. + progress_dialog.set_action("Installing python dependencies with pip") + # Install dependencies QgsMessageLog.logMessage(f"Installing python dependencies from {REQUIREMENTS_PATH}", "QGEP") _run_cmd(f'pip install -r {REQUIREMENTS_PATH}', error_message='Could not install python dependencies') - # Refresh paths - importlib.reload(site) - def get_current_version(pgservice): if not os.path.exists(DATAMODEL_DELTAS_DIR): @@ -172,43 +156,72 @@ def get_available_versions(): if not f.startswith('delta_'): continue parts = f.split('_') + version = parts[1] + if version > MAIN_DATAMODEL_RELEASE: + # We don't want to list versions that are not yet released + continue versions.add(parts[1]) return sorted(list(versions)) -def upgrade_version(pgservice, version, srid): +def upgrade_version(pgservice, version, srid, progress_dialog): try: # If we can't get current version, it's probably that the DB is not initialized # (or maybe we can't connect, but we can't know easily with PUM) + get_current_version(pgservice) except QGEPDatamodelError: + progress_dialog.set_action("Initializing the datamodel") + QgsMessageLog.logMessage("Upgrading failed, trying to initialize the datamodel", "QGEP") - feedback = QgsFeedback() # TODO : this should be done by PUM directly (see https://github.com/opengisch/pum/issues/94) # also currently SRID doesn't work try: - sql_path = _download(DATAMODEL_STRUCTURE_URL.format(version=version), f"structure_with_value_lists-{version}.sql", feedback) - conn = psycopg2.connect(f"service={pgservice}") + progress_dialog.set_action("Downloading the structure script") + sql_path = _download(INITIAL_STRUCTURE_URL_TEMPLATE.format(version=version), f"structure_with_value_lists-{version}-{srid}.sql") + + # Dirty hack to customize SRID in a dump + if srid != '2056': + with open(sql_path, 'r') as file : + contents = file.read() + contents = contents.replace('2056', srid) + with open(sql_path, 'w') as file: + file.write(contents) + + try: + conn = psycopg2.connect(f"service={pgservice}") + except psycopg2.Error as e: + # If may be that the database doesn't exist yet + progress_dialog.set_action("Creating the database") + dbname = get_pgservice_database_name(pgservice) + _run_cmd( + f'psql -c "CREATE DATABASE {dbname};" "service={pgservice} dbname=postgres"', + error_message='Errors when initializing the database. Consult logs for more information.' + ) + conn = psycopg2.connect(f"service={pgservice}") + + progress_dialog.set_action("Running the initialization scripts") cur = conn.cursor() cur.execute('CREATE SCHEMA IF NOT EXISTS qgep_sys;') cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;') - cur.execute(open(sql_path, "r").read()) + # we cannot use this, as it doesn't support COPY statements + # this means we'll run through psql without transaction :-/ + # cur.execute(open(sql_path, "r").read()) conn.commit() cur.close() conn.close() + _run_cmd( + f'psql -f {sql_path} "service={pgservice}"', + error_message='Errors when initializing the database. Consult logs for more information.' + ) + except psycopg2.Error as e: raise QGEPDatamodelError(str(e)) - # TODO : this also should be done by pum upgrade directly (see https://github.com/opengisch/pum/issues/94) - _run_cmd( - f'pum baseline -p {pgservice} -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR} -b 0.0.0', - cwd=os.path.dirname(DATAMODEL_DELTAS_DIR), - error_message='Errors when initializing the database. Consult logs for more information.' - ) - + progress_dialog.set_action("Running pum upgrade") return _run_cmd( f'pum upgrade -p {pgservice} -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR} -u {version} -v int SRID {srid}', cwd=os.path.dirname(DATAMODEL_DELTAS_DIR), @@ -217,6 +230,12 @@ def upgrade_version(pgservice, version, srid): def load_project(pgservice): + + # Unzip QGEP + qgep_path = _download(QGEP_RELEASE_URL, 'qgep.zip') + qgep_zip = zipfile.ZipFile(qgep_path) + qgep_zip.extractall(TEMP_DIR) + with open(QGIS_PROJECT_PATH, 'r') as original_project: contents = original_project.read() @@ -241,6 +260,11 @@ def get_pgservice_configs_names(): return config.sections() +def get_pgservice_database_name(service_name): + config = read_pgservice() + return config[service_name]['dbname'] + + def write_pgservice_conf(service_name, config_dict): config = read_pgservice() config[service_name] = config_dict @@ -252,7 +276,8 @@ class EqualsSpaceRemover: def __init__(self, output_file): self.output_file = output_file - def write(self, what): - self.output_file.write(what.replace(" = ", "=", 1)) + def write(self, content): + content = content.replace(" = ", "=", 1) + self.output_file.write(content.encode('utf-8')) - config.write(EqualsSpaceRemover(open(PG_CONFIG_PATH, 'w'))) \ No newline at end of file + config.write(EqualsSpaceRemover(open(PG_CONFIG_PATH, 'wb'))) \ No newline at end of file diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 517c7cae..e4e68eeb 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -28,8 +28,9 @@ import functools from builtins import str -from qgis.PyQt.QtWidgets import QDialog, QMessageBox -from qgis.core import QgsSettings, QgsMessageLog +from qgis.PyQt.QtCore import Qt +from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QProgressDialog , QApplication +from qgis.core import QgsSettings, QgsMessageLog, QgsFeedback from ..utils import get_ui_class from .. import datamodel_initializer @@ -49,6 +50,20 @@ def wrapper(*args, **kwargs): err.exec_() return wrapper +class ProgressDialog(QProgressDialog): + def __init__(self, label, **kwargs): + super().__init__(label, "Cancel", 0, 100) + self.setRange(0, 0) + self.setLabelText("Starting...") + self.show() + + def set_progress(self, progress): + self.setValue(int(progress)) + QApplication.processEvents() + + def set_action(self, text): + self.setLabelText(text) + QApplication.processEvents() class QgepPgserviceEditorDialog(QDialog, get_ui_class('qgeppgserviceeditordialog.ui')): @@ -113,11 +128,13 @@ def showEvent(self, event): def enable_buttons_if_ready(self): self.versionUpgradeButton.setEnabled(all(self.checks.values())) - self.loadProjectButton.setEnabled(self.checks['release']) + self.loadProjectButton.setEnabled(self.checks['release'] and self.checks['pgconfig']) @qgep_datamodel_error_catcher def install_deps(self): - datamodel_initializer.install_deps() + progress_dialog = ProgressDialog("Installing dependencies") + + datamodel_initializer.install_deps(progress_dialog) self.update_requirements_checks() @qgep_datamodel_error_catcher @@ -204,7 +221,7 @@ def update_versions_checks(self): if current_version is None or current_version == '0.0.0' or current_version in available_versions: self.checks['current_version'] = True self.versionCheckLabel.setText(current_version or 'not initialized') - self.versionCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + self.versionCheckLabel.setStyleSheet('color: rgb(170, 65, 0);\nfont-weight: bold;') elif current_version is None: self.versionCheckLabel.setText("could not determine version") self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') @@ -233,10 +250,18 @@ def upgrade_version(self): confirm.setStandardButtons(QMessageBox.Apply | QMessageBox.Cancel) confirm.setIcon(QMessageBox.Warning) - if confirm.exec_() == QMessageBox.Apply: - datamodel_initializer.upgrade_version(pgservice, version, self.sridLineEdit.text()) + if confirm.exec_() == QMessageBox.Apply: + progress_dialog = ProgressDialog("Upgrading the datamodel") + + srid = self.sridLineEdit.text() + datamodel_initializer.upgrade_version(pgservice, version, srid, progress_dialog) self.update_versions_checks() + success = QMessageBox() + success.setText("Datamodel successfully upgraded") + success.setIcon(QMessageBox.Success) + success.exec_() + @qgep_datamodel_error_catcher def load_project(self): pgservice = self.pgserviceComboBox.currentText() From 8adc1a942098f7e82971029d91eacecec020799d Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Wed, 28 Oct 2020 20:13:15 +0100 Subject: [PATCH 05/35] avancement --- qgepplugin/datamodel_initializer/__init__.py | 283 ---------- qgepplugin/gui/qgepdatamodeldialog.py | 532 +++++++++++++++---- qgepplugin/ui/qgepdatamodeldialog.ui | 117 ++-- 3 files changed, 483 insertions(+), 449 deletions(-) delete mode 100644 qgepplugin/datamodel_initializer/__init__.py diff --git a/qgepplugin/datamodel_initializer/__init__.py b/qgepplugin/datamodel_initializer/__init__.py deleted file mode 100644 index 84dce48e..00000000 --- a/qgepplugin/datamodel_initializer/__init__.py +++ /dev/null @@ -1,283 +0,0 @@ -import os -import tempfile -import hashlib -import zipfile -import subprocess -import pkg_resources -import site -import importlib -import configparser -import psycopg2 - -from qgis.PyQt.QtCore import QUrl, QFile, QIODevice -from qgis.PyQt.QtNetwork import QNetworkRequest -from qgis.core import QgsNetworkAccessManager, QgsMessageLog, Qgis, QgsProject - - -# Basic config -MAIN_DATAMODEL_RELEASE = '1.5.2' -QGEP_RELEASE = '8.0' -TEMP_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', 'datamodel-init') - -# Path for pg_service.conf -if os.environ.get('PGSERVICEFILE'): - PG_CONFIG_PATH = os.environ.get('PGSERVICEFILE') -elif os.environ.get('PGSYSCONFDIR'): - PG_CONFIG_PATH = os.path.join(os.environ.get('PGSYSCONFDIR'), 'pg_service.conf') -else: - PG_CONFIG_PATH = ' ~/.pg_service.conf' - -# Derived urls/paths, may require adaptations if release structure changes -MAIN_DATAMODEL_RELEASE_URL = f'https://github.com/QGEP/datamodel/archive/{MAIN_DATAMODEL_RELEASE}.zip' -QGEP_RELEASE_URL = f'https://github.com/QGEP/QGEP/releases/download/v{QGEP_RELEASE}/qgep.zip' # TODO : we should download the release according the the current datamodel version, not the latest one -REQUIREMENTS_PATH = os.path.join(TEMP_DIR, f'datamodel-{MAIN_DATAMODEL_RELEASE}', 'requirements.txt') -DATAMODEL_DELTAS_DIR = os.path.join(TEMP_DIR, f'datamodel-{MAIN_DATAMODEL_RELEASE}', "delta") -QGIS_PROJECT_PATH = os.path.join(TEMP_DIR, 'project', 'qgep.qgs') -INITIAL_STRUCTURE_URL_TEMPLATE = "https://github.com/QGEP/datamodel/releases/download/{version}/qgep_v{version}_structure_with_value_lists.sql" - - -class QGEPDatamodelError(Exception): - pass - - -def _run_cmd(shell_command, cwd=None, error_message='Subprocess error, see logs for more information'): - """ - Helper to run commands through subprocess - """ - QgsMessageLog.logMessage(f"Running command : {shell_command}", "QGEP") - result = subprocess.run(shell_command, cwd=cwd, shell=True, capture_output=True) - if result.stdout: - QgsMessageLog.logMessage(result.stdout.decode(), "QGEP") - if result.stderr: - QgsMessageLog.logMessage(result.stderr.decode(), "QGEP", level=Qgis.Critical) - if result.returncode: - raise QGEPDatamodelError(error_message) - return result.stdout.decode() - - -def _download(url, filename): - os.makedirs(TEMP_DIR, exist_ok=True) - - network_manager = QgsNetworkAccessManager.instance() - QgsMessageLog.logMessage(f"Downloading {url} to {TEMP_DIR}", "QGEP") - reply = network_manager.blockingGet(QNetworkRequest(QUrl(url))) - download_path = os.path.join(TEMP_DIR, filename) - download_file = QFile(download_path) - download_file.open(QIODevice.WriteOnly) - download_file.write(reply.content()) - download_file.close() - return download_file.fileName() - -def check_release_exists(): - return os.path.exists(REQUIREMENTS_PATH) and os.path.exists(DATAMODEL_DELTAS_DIR) - -def check_python_requirements(): - return len(missing_python_requirements()) == 0 - -def missing_python_requirements(): - # see https://stackoverflow.com/a/45474387/13690651 - - missing = [] - if not check_release_exists(): - missing.append(('unknown', 'requirements not found')) - else: - requirements = pkg_resources.parse_requirements(open(REQUIREMENTS_PATH)) - for requirement in requirements: - try: - pkg_resources.require(str(requirement)) - except pkg_resources.DistributionNotFound: - missing.append((requirement, 'missing')) - except pkg_resources.VersionConflict: - missing.append((requirement, 'conflict')) - return missing - -def install_deps(progress_dialog): - - # Download the files if needed - - progress_dialog.set_action("Downloading the release") - - if not check_release_exists(): - # Download files - datamodel_path = _download(MAIN_DATAMODEL_RELEASE_URL, 'datamodel.zip') - - # Unzip - QgsMessageLog.logMessage(f"Extracting files to {TEMP_DIR}", "QGEP") - datamodel_zip = zipfile.ZipFile(datamodel_path) - datamodel_zip.extractall(TEMP_DIR) - - # TODO : Ideally, this should be done in a venv, as to avoid permission issues and/or modification - # of libraries versions that could affect other parts of the system. - # We could initialize a venv in the user's directory, and activate it. - # It's almost doable when only running commands from the command line (in which case we could - # just prepent something like `path/to/venv/Scripts/activate && ` to commands, /!\ syntax differs on Windows), - # but to be really useful, it would be best to then enable the virtualenv from within python directly. - # It seems venv doesn't provide a way to do so, while virtualenv does - # (see https://stackoverflow.com/a/33637378/13690651) - # but virtualenv isn't in the stdlib... So we'd have to install it globally ! Argh... - # Anyway, pip deps support should be done in QGIS one day so all plugins can benefit. - # In the mean time we just install globally and hope for the best. - - progress_dialog.set_action("Installing python dependencies with pip") - - # Install dependencies - QgsMessageLog.logMessage(f"Installing python dependencies from {REQUIREMENTS_PATH}", "QGEP") - _run_cmd(f'pip install -r {REQUIREMENTS_PATH}', error_message='Could not install python dependencies') - - -def get_current_version(pgservice): - if not os.path.exists(DATAMODEL_DELTAS_DIR): - return None - - pum_info = _run_cmd( - f'pum info -p {pgservice} -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR}', - error_message='Could not get current version, are you sure the database is accessible ?' - ) - for line in pum_info.splitlines(): - line = line.strip() - if not line: - continue - parts = line.split('|') - if len(parts) > 1: - version = parts[1].strip() - return version - - -def get_available_versions(): - # TODO : this should be done by PUM directly (see https://github.com/opengisch/pum/issues/94) - - if not os.path.exists(DATAMODEL_DELTAS_DIR): - return [] - - versions = set() - for f in os.listdir(DATAMODEL_DELTAS_DIR): - if not os.path.isfile(os.path.join(DATAMODEL_DELTAS_DIR, f)): - continue - if not f.startswith('delta_'): - continue - parts = f.split('_') - version = parts[1] - if version > MAIN_DATAMODEL_RELEASE: - # We don't want to list versions that are not yet released - continue - versions.add(parts[1]) - return sorted(list(versions)) - - -def upgrade_version(pgservice, version, srid, progress_dialog): - try: - # If we can't get current version, it's probably that the DB is not initialized - # (or maybe we can't connect, but we can't know easily with PUM) - - get_current_version(pgservice) - except QGEPDatamodelError: - - progress_dialog.set_action("Initializing the datamodel") - - QgsMessageLog.logMessage("Upgrading failed, trying to initialize the datamodel", "QGEP") - - - # TODO : this should be done by PUM directly (see https://github.com/opengisch/pum/issues/94) - # also currently SRID doesn't work - try: - progress_dialog.set_action("Downloading the structure script") - sql_path = _download(INITIAL_STRUCTURE_URL_TEMPLATE.format(version=version), f"structure_with_value_lists-{version}-{srid}.sql") - - # Dirty hack to customize SRID in a dump - if srid != '2056': - with open(sql_path, 'r') as file : - contents = file.read() - contents = contents.replace('2056', srid) - with open(sql_path, 'w') as file: - file.write(contents) - - try: - conn = psycopg2.connect(f"service={pgservice}") - except psycopg2.Error as e: - # If may be that the database doesn't exist yet - progress_dialog.set_action("Creating the database") - dbname = get_pgservice_database_name(pgservice) - _run_cmd( - f'psql -c "CREATE DATABASE {dbname};" "service={pgservice} dbname=postgres"', - error_message='Errors when initializing the database. Consult logs for more information.' - ) - conn = psycopg2.connect(f"service={pgservice}") - - progress_dialog.set_action("Running the initialization scripts") - cur = conn.cursor() - cur.execute('CREATE SCHEMA IF NOT EXISTS qgep_sys;') - cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;') - # we cannot use this, as it doesn't support COPY statements - # this means we'll run through psql without transaction :-/ - # cur.execute(open(sql_path, "r").read()) - conn.commit() - cur.close() - conn.close() - _run_cmd( - f'psql -f {sql_path} "service={pgservice}"', - error_message='Errors when initializing the database. Consult logs for more information.' - ) - - except psycopg2.Error as e: - raise QGEPDatamodelError(str(e)) - - progress_dialog.set_action("Running pum upgrade") - return _run_cmd( - f'pum upgrade -p {pgservice} -t qgep_sys.pum_info -d {DATAMODEL_DELTAS_DIR} -u {version} -v int SRID {srid}', - cwd=os.path.dirname(DATAMODEL_DELTAS_DIR), - error_message='Errors when upgrading the database. Consult logs for more information.' - ) - - -def load_project(pgservice): - - # Unzip QGEP - qgep_path = _download(QGEP_RELEASE_URL, 'qgep.zip') - qgep_zip = zipfile.ZipFile(qgep_path) - qgep_zip.extractall(TEMP_DIR) - - with open(QGIS_PROJECT_PATH, 'r') as original_project: - contents = original_project.read() - - # replace the service name - contents = contents.replace("service='pg_qgep'", f"service='{pgservice}'") - - output_file = tempfile.NamedTemporaryFile(suffix='.qgs', delete=False) - output_file.write(contents.encode('utf8')) - - QgsProject.instance().read(output_file.name) - - -def read_pgservice(): - config = configparser.ConfigParser() - if os.path.exists(PG_CONFIG_PATH): - config.read(PG_CONFIG_PATH) - return config - - -def get_pgservice_configs_names(): - config = read_pgservice() - return config.sections() - - -def get_pgservice_database_name(service_name): - config = read_pgservice() - return config[service_name]['dbname'] - - -def write_pgservice_conf(service_name, config_dict): - config = read_pgservice() - config[service_name] = config_dict - - class EqualsSpaceRemover: - # see https://stackoverflow.com/a/25084055/13690651 - output_file = None - - def __init__(self, output_file): - self.output_file = output_file - - def write(self, content): - content = content.replace(" = ", "=", 1) - self.output_file.write(content.encode('utf-8')) - - config.write(EqualsSpaceRemover(open(PG_CONFIG_PATH, 'wb'))) \ No newline at end of file diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index e4e68eeb..a571ee48 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -26,14 +26,19 @@ import os import configparser import functools +import zipfile +import tempfile +import pkg_resources +import subprocess +import psycopg2 -from builtins import str -from qgis.PyQt.QtCore import Qt -from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QProgressDialog , QApplication -from qgis.core import QgsSettings, QgsMessageLog, QgsFeedback +from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QProgressDialog, QApplication +from qgis.PyQt.QtCore import QUrl, QFile, QIODevice +from qgis.PyQt.QtNetwork import QNetworkRequest + +from qgis.core import QgsMessageLog, QgsNetworkAccessManager, Qgis, QgsProject from ..utils import get_ui_class -from .. import datamodel_initializer def qgep_datamodel_error_catcher(func): @@ -43,27 +48,13 @@ def qgep_datamodel_error_catcher(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except datamodel_initializer.QGEPDatamodelError as e: - err = QMessageBox() - err.setText(str(e)) - err.setIcon(QMessageBox.Warning) - err.exec_() + except QGEPDatamodelError as e: + args[0]._show_error(str(e)) return wrapper -class ProgressDialog(QProgressDialog): - def __init__(self, label, **kwargs): - super().__init__(label, "Cancel", 0, 100) - self.setRange(0, 0) - self.setLabelText("Starting...") - self.show() - - def set_progress(self, progress): - self.setValue(int(progress)) - QApplication.processEvents() - def set_action(self, text): - self.setLabelText(text) - QApplication.processEvents() +class QGEPDatamodelError(Exception): + pass class QgepPgserviceEditorDialog(QDialog, get_ui_class('qgeppgserviceeditordialog.ui')): @@ -95,148 +86,378 @@ def conf_dict(self): } - class QgepDatamodelInitToolDialog(QDialog, get_ui_class('qgepdatamodeldialog.ui')): + AVAILABLE_VERSIONS = { + # 'master': '8.0', # too dangerous to expose here + '1.5.2': '8.0', + '1.5.1': '8.0', + '1.5.0': '8.0', + '1.4.0': '7.0', + } + TEMP_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', 'datamodel-init') + + # Path for pg_service.conf + if os.environ.get('PGSERVICEFILE'): + PG_CONFIG_PATH = os.environ.get('PGSERVICEFILE') + elif os.environ.get('PGSYSCONFDIR'): + PG_CONFIG_PATH = os.path.join(os.environ.get('PGSYSCONFDIR'), 'pg_service.conf') + else: + PG_CONFIG_PATH = ' ~/.pg_service.conf' + + MAIN_DATAMODEL_RELEASE = '1.5.2' + QGEP_RELEASE = '8.0' + + # Derived urls/paths, may require adaptations if release structure changes + DATAMODEL_URL_TEMPLATE = 'https://github.com/QGEP/datamodel/archive/{}.zip' + REQUIREMENTS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'requirements.txt') + DELTAS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'delta') + INIT_SCRIPT_URL_TEMPLATE = "https://github.com/QGEP/datamodel/releases/download/{}/qgep_v{}_structure_with_value_lists.sql" + QGEP_PROJECT_URL_TEMPLATE = 'https://github.com/QGEP/QGEP/releases/download/v{}/qgep.zip' + QGEP_PROJECT_PATH_TEMPLATE = os.path.join(TEMP_DIR, "project", 'qgep.qgs') + def __init__(self, parent=None): QDialog.__init__(self, parent) self.setupUi(self) - self.installDepsButton.pressed.connect(self.install_deps) + self.progress_dialog = None + + # Populate the versions + self.targetVersionComboBox.clear() + self.targetVersionComboBox.addItem('- SELECT TARGET VERSION -') + self.targetVersionComboBox.model().item(0).setEnabled(False) + for version in sorted(list(self.AVAILABLE_VERSIONS.keys()), reverse=True): + self.targetVersionComboBox.addItem(version) + + # Show the pgconfig path + self.pgservicePathLabel.setText(self.PG_CONFIG_PATH) + + # Connect some signals + + self.targetVersionComboBox.activated.connect(self.switch_datamodel) - self.pgserviceComboBox.activated.connect(self.update_pgconfig_checks) + self.installDepsButton.pressed.connect(self.install_requirements) + + self.pgserviceComboBox.activated.connect(self.select_pgconfig) self.pgserviceAddButton.pressed.connect(self.add_pgconfig) self.versionUpgradeButton.pressed.connect(self.upgrade_version) self.loadProjectButton.pressed.connect(self.load_project) + # Initialize the checks self.checks = { - 'release': False, - 'dependencies': False, + 'datamodel': False, + 'requirements': False, 'pgconfig': False, 'current_version': False, } - @qgep_datamodel_error_catcher + # Properties + + @property + def version(self): + return self.targetVersionComboBox.currentText() + + @property + def conf(self): + return self.pgserviceComboBox.currentText() + + # Feedback helpers + + def _show_progress(self, message): + if self.progress_dialog is None: + self.progress_dialog = QProgressDialog("Starting...", "Cancel", 0, 0) + self.progress_dialog.setLabelText(message) + self.progress_dialog.show() + QApplication.processEvents() + + def _done_progress(self): + self.progress_dialog.close() + self.progress_dialog.deleteLater() + self.progress_dialog = None + QApplication.processEvents() + + def _show_error(self, message): + self._done_progress() + err = QMessageBox() + err.setText(message) + err.setIcon(QMessageBox.Warning) + err.exec_() + + # Actions helpers + + def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see logs for more information'): + """ + Helper to run commands through subprocess + """ + QgsMessageLog.logMessage(f"Running command : {shell_command}", "QGEP") + result = subprocess.run(shell_command, cwd=cwd, shell=True, capture_output=True) + if result.stdout: + QgsMessageLog.logMessage(result.stdout.decode(), "QGEP") + if result.stderr: + QgsMessageLog.logMessage(result.stderr.decode(), "QGEP", level=Qgis.Critical) + if result.returncode: + raise QGEPDatamodelError(f"{error_message}\n{result.stdout.decode()}\n{result.stderr.decode()}") + return result.stdout.decode() + + def _download(self, url, filename): + os.makedirs(self.TEMP_DIR, exist_ok=True) + + network_manager = QgsNetworkAccessManager.instance() + reply = network_manager.blockingGet(QNetworkRequest(QUrl(url))) + download_path = os.path.join(self.TEMP_DIR, filename) + download_file = QFile(download_path) + download_file.open(QIODevice.WriteOnly) + download_file.write(reply.content()) + download_file.close() + return download_file.fileName() + + def _read_pgservice(self): + config = configparser.ConfigParser() + if os.path.exists(self.PG_CONFIG_PATH): + config.read(self.PG_CONFIG_PATH) + return config + + def _get_pgservice_configs_names(self): + config = self._read_pgservice() + return config.sections() + + def _write_pgservice_conf(self, service_name, config_dict): + config = self._read_pgservice() + config[service_name] = config_dict + + class EqualsSpaceRemover: + # see https://stackoverflow.com/a/25084055/13690651 + output_file = None + + def __init__(self, output_file): + self.output_file = output_file + + def write(self, content): + content = content.replace(" = ", "=", 1) + self.output_file.write(content.encode('utf-8')) + + config.write(EqualsSpaceRemover(open(self.PG_CONFIG_PATH, 'wb'))) + + def _get_current_version(self): + # Dirty parsing of pum info + deltas_dir = self.DELTAS_PATH_TEMPLATE.format(self.version) + if not os.path.exists(deltas_dir): + return None + + pum_info = self._run_cmd( + f'pum info -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir}', + error_message='Could not get current version, are you sure the database is accessible ?' + ) + for line in pum_info.splitlines(): + line = line.strip() + if not line: + continue + parts = line.split('|') + if len(parts) > 1: + version = parts[1].strip() + return version + + # Display + def showEvent(self, event): - self.refresh_pgservice_combobox() - self.update_requirements_checks() - self.update_pgconfig_checks() - self.update_versions_checks() - self.pgservicePathLabel.setText(datamodel_initializer.PG_CONFIG_PATH) + self.update_pgconfig_combobox() + self.check_datamodel() + self.check_requirements() + self.check_pgconfig() + self.check_version() super().showEvent(event) def enable_buttons_if_ready(self): self.versionUpgradeButton.setEnabled(all(self.checks.values())) - self.loadProjectButton.setEnabled(self.checks['release'] and self.checks['pgconfig']) + self.installDepsButton.setEnabled(self.checks['datamodel'] and not self.checks['requirements']) + self.loadProjectButton.setEnabled(self.checks['datamodel'] and self.checks['pgconfig']) - @qgep_datamodel_error_catcher - def install_deps(self): - progress_dialog = ProgressDialog("Installing dependencies") + # Datamodel - datamodel_initializer.install_deps(progress_dialog) - self.update_requirements_checks() + def check_datamodel(self): + requirements_exists = os.path.exists(self.REQUIREMENTS_PATH_TEMPLATE.format(self.version)) + deltas_exists = os.path.exists(self.DELTAS_PATH_TEMPLATE.format(self.version)) - @qgep_datamodel_error_catcher - def update_requirements_checks(self): + check = requirements_exists and deltas_exists - if datamodel_initializer.check_release_exists(): - self.checks['release'] = True + if check: self.releaseCheckLabel.setText('ok') self.releaseCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') else: - self.checks['release'] = False self.releaseCheckLabel.setText('not found') self.releaseCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') - if datamodel_initializer.check_python_requirements(): - self.checks['dependencies'] = True + self.checks['datamodel'] = check + self.enable_buttons_if_ready() + + return check + + @qgep_datamodel_error_catcher + def switch_datamodel(self, _=None): + # Download the datamodel if it doesn't exist + + if not self.check_datamodel(): + + self._show_progress("Downloading the release") + + # Download files + datamodel_path = self._download(self.DATAMODEL_URL_TEMPLATE.format(self.version), 'datamodel.zip') + + # Unzip + datamodel_zip = zipfile.ZipFile(datamodel_path) + datamodel_zip.extractall(self.TEMP_DIR) + + # Cleanup + # os.unlink(datamodel_path) + + # Update UI + self.check_datamodel() + + self._done_progress() + + self.check_requirements() + self.check_version() + + # Requirements + + def check_requirements(self): + + missing = [] + if not self.check_datamodel(): + missing.append(('unknown', 'no datamodel')) + else: + requirements = pkg_resources.parse_requirements(open(self.REQUIREMENTS_PATH_TEMPLATE.format(self.version))) + for requirement in requirements: + try: + pkg_resources.require(str(requirement)) + except pkg_resources.DistributionNotFound: + missing.append((requirement, 'missing')) + except pkg_resources.VersionConflict: + missing.append((requirement, 'conflict')) + + check = len(missing) == 0 + + if check: self.pythonCheckLabel.setText('ok') self.pythonCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') else: - self.checks['dependencies'] = False - errors = datamodel_initializer.missing_python_requirements() - self.pythonCheckLabel.setText('\n'.join(f'{dep}: {err}' for dep, err in errors)) + self.pythonCheckLabel.setText('\n'.join(f'{dep}: {err}' for dep, err in missing)) self.pythonCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.checks['requirements'] = check self.enable_buttons_if_ready() - @qgep_datamodel_error_catcher - def refresh_pgservice_combobox(self): - self.pgserviceComboBox.clear() - config_names = datamodel_initializer.get_pgservice_configs_names() - for config_name in config_names: - self.pgserviceComboBox.addItem(config_name) - self.pgserviceComboBox.setCurrentIndex(-1) + return check @qgep_datamodel_error_catcher - def add_pgconfig(self): - existing_config_names = datamodel_initializer.get_pgservice_configs_names() - add_dialog = QgepPgserviceEditorDialog(existing_config_names) - if add_dialog.exec_() == QDialog.Accepted: - name = add_dialog.conf_name() - conf = add_dialog.conf_dict() - datamodel_initializer.write_pgservice_conf(name, conf) - self.refresh_pgservice_combobox() - self.pgserviceComboBox.setCurrentIndex(self.pgserviceComboBox.findText(name)) - self.update_pgconfig_checks() + def install_requirements(self): - @qgep_datamodel_error_catcher - def update_pgconfig_checks(self, _=None): + # TODO : Ideally, this should be done in a venv, as to avoid permission issues and/or modification + # of libraries versions that could affect other parts of the system. + # We could initialize a venv in the user's directory, and activate it. + # It's almost doable when only running commands from the command line (in which case we could + # just prepent something like `path/to/venv/Scripts/activate && ` to commands, /!\ syntax differs on Windows), + # but to be really useful, it would be best to then enable the virtualenv from within python directly. + # It seems venv doesn't provide a way to do so, while virtualenv does + # (see https://stackoverflow.com/a/33637378/13690651) + # but virtualenv isn't in the stdlib... So we'd have to install it globally ! Argh... + # Anyway, pip deps support should be done in QGIS one day so all plugins can benefit. + # In the mean time we just install globally and hope for the best. + + self._show_progress("Installing python dependencies with pip") + + # Install dependencies + requirements_file_path = self.REQUIREMENTS_PATH_TEMPLATE.format(self.version) + QgsMessageLog.logMessage(f"Installing python dependencies from {requirements_file_path}", "QGEP") + self._run_cmd(f'pip install -r {requirements_file_path}', error_message='Could not install python dependencies') + + self._done_progress() + + # Update UI + self.check_requirements() - if self.pgserviceComboBox.currentText(): - self.checks['pgconfig'] = True + # Pgservice + + def check_pgconfig(self): + + check = self.pgserviceComboBox.currentText() != '' + if check: self.pgconfigCheckLabel.setText('ok') self.pgconfigCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') else: - self.checks['pgconfig'] = False self.pgconfigCheckLabel.setText('not set') self.pgconfigCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') - self.update_versions_checks() + self.checks['pgconfig'] = check + self.enable_buttons_if_ready() - @qgep_datamodel_error_catcher - def update_versions_checks(self): - self.checks['current_version'] = False + return check - available_versions = datamodel_initializer.get_available_versions() - self.targetVersionComboBox.clear() - for version in reversed(available_versions): - self.targetVersionComboBox.addItem(version) + def add_pgconfig(self): + existing_config_names = self._get_pgservice_configs_names() + add_dialog = QgepPgserviceEditorDialog(existing_config_names) + if add_dialog.exec_() == QDialog.Accepted: + name = add_dialog.conf_name() + conf = add_dialog.conf_dict() + self._write_pgservice_conf(name, conf) + self.update_pgconfig_combobox() + self.pgserviceComboBox.setCurrentIndex(self.pgserviceComboBox.findText(name)) + self.select_pgconfig() + + def update_pgconfig_combobox(self): + self.pgserviceComboBox.clear() + config_names = self._get_pgservice_configs_names() + for config_name in config_names: + self.pgserviceComboBox.addItem(config_name) + self.pgserviceComboBox.setCurrentIndex(0) + + def select_pgconfig(self, _=None): + self.check_pgconfig() + self.check_version() + + # Version + + def check_version(self): + check = False pgservice = self.pgserviceComboBox.currentText() if not pgservice: self.versionCheckLabel.setText('service not selected') self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') - return - try: - current_version = datamodel_initializer.get_current_version(pgservice) - except datamodel_initializer.QGEPDatamodelError: - # Can happend if PUM is not initialized, unfortunately we can't really - # determine if this is a connection error or if PUM is not initailized - # see https://github.com/opengisch/pum/issues/96 - current_version = None - - if current_version is None or current_version == '0.0.0' or current_version in available_versions: - self.checks['current_version'] = True - self.versionCheckLabel.setText(current_version or 'not initialized') - self.versionCheckLabel.setStyleSheet('color: rgb(170, 65, 0);\nfont-weight: bold;') - elif current_version is None: - self.versionCheckLabel.setText("could not determine version") - self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') else: - self.versionCheckLabel.setText(f"invalid version : {current_version}") - self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') - - # disable unapplicable versions - for i in range(self.targetVersionComboBox.model().rowCount()): - item_version = self.targetVersionComboBox.model().item(i).text() - enabled = current_version is None or item_version >= current_version - self.targetVersionComboBox.model().item(i).setEnabled(enabled) + try: + current_version = self._get_current_version() + except QGEPDatamodelError: + # Can happend if PUM is not initialized, unfortunately we can't really + # determine if this is a connection error or if PUM is not initailized + # see https://github.com/opengisch/pum/issues/96 + current_version = None + + if current_version is None: + check = True + self.versionCheckLabel.setText('not initialized') + self.versionCheckLabel.setStyleSheet('color: rgb(170, 65, 0);\nfont-weight: bold;') + elif current_version <= self.version: + check = True + self.versionCheckLabel.setText(current_version) + self.versionCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + elif current_version > self.version: + check = False + self.versionCheckLabel.setText(f"{current_version} (cannot downgrade)") + self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + else: + check = False + self.versionCheckLabel.setText(f"{current_version} (invalid version)") + self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + + self.checks['current_version'] = check self.enable_buttons_if_ready() + return check + @qgep_datamodel_error_catcher def upgrade_version(self): version = self.targetVersionComboBox.currentText() @@ -250,19 +471,108 @@ def upgrade_version(self): confirm.setStandardButtons(QMessageBox.Apply | QMessageBox.Cancel) confirm.setIcon(QMessageBox.Warning) - if confirm.exec_() == QMessageBox.Apply: - progress_dialog = ProgressDialog("Upgrading the datamodel") + if confirm.exec_() == QMessageBox.Apply: + + self._show_progress("Upgrading the datamodel") srid = self.sridLineEdit.text() - datamodel_initializer.upgrade_version(pgservice, version, srid, progress_dialog) - self.update_versions_checks() + + try: + current_version = self._get_current_version() + except QGEPDatamodelError: + # Can happend if PUM is not initialized, unfortunately we can't really + # determine if this is a connection error or if PUM is not initailized + # see https://github.com/opengisch/pum/issues/96 + current_version = None + + if current_version is None: + # If we can't get current version, it's probably that the DB is not initialized + # (or maybe we can't connect, but we can't know easily with PUM) + + self._show_progress("Initializing the datamodel") + + # TODO : this should be done by PUM directly (see https://github.com/opengisch/pum/issues/94) + # also currently SRID doesn't work + try: + self._show_progress("Downloading the structure script") + url = self.INIT_SCRIPT_URL_TEMPLATE.format(self.version, self.version) + sql_path = self._download(url, f"structure_with_value_lists-{self.version}-{srid}.sql") + + # Dirty hack to customize SRID in a dump + if srid != '2056': + with open(sql_path, 'r') as file: + contents = file.read() + contents = contents.replace('2056', srid) + with open(sql_path, 'w') as file: + file.write(contents) + + try: + conn = psycopg2.connect(f"service={self.conf}") + except psycopg2.Error: + # It may be that the database doesn't exist yet + # in that case, we try to connect to the postgres database and to create it from there + self._show_progress("Creating the database") + dbname = self._read_pgservice()[self.conf]['dbname'] + self._run_cmd( + f'psql -c "CREATE DATABASE {dbname};" "service={self.conf} dbname=postgres"', + error_message='Errors when initializing the database.' + ) + conn = psycopg2.connect(f"service={self.conf}") + + self._show_progress("Running the initialization scripts") + cur = conn.cursor() + cur.execute('CREATE SCHEMA IF NOT EXISTS qgep_sys;') + cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;') + # we cannot use this, as it doesn't support COPY statements + # this means we'll run through psql without transaction :-/ + # cur.execute(open(sql_path, "r").read()) + conn.commit() + cur.close() + conn.close() + self._run_cmd( + f'psql -f {sql_path} "service={self.conf}"', + error_message='Errors when initializing the database.' + ) + + except psycopg2.Error as e: + raise QGEPDatamodelError(str(e)) + + self._show_progress("Running pum upgrade") + deltas_dir = self.DELTAS_PATH_TEMPLATE.format(self.version) + return self._run_cmd( + f'pum upgrade -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir} -u {self.version} -v int SRID {srid}', + cwd=os.path.dirname(deltas_dir), + error_message='Errors when upgrading the database.' + ) + + self.check_version() + + self._done_progress() success = QMessageBox() success.setText("Datamodel successfully upgraded") success.setIcon(QMessageBox.Success) success.exec_() - + + # Project + @qgep_datamodel_error_catcher def load_project(self): - pgservice = self.pgserviceComboBox.currentText() - datamodel_initializer.load_project(pgservice) + + # Unzip QGEP + qgep_release = self.AVAILABLE_VERSIONS[self.version] + url = self.QGEP_PROJECT_URL_TEMPLATE.format(qgep_release) + qgep_path = self._download(url, 'qgep.zip') + qgep_zip = zipfile.ZipFile(qgep_path) + qgep_zip.extractall(self.TEMP_DIR) + + with open(self.QGEP_PROJECT_PATH_TEMPLATE, 'r') as original_project: + contents = original_project.read() + + # replace the service name + contents = contents.replace("service='pg_qgep'", f"service='{self.conf}'") + + output_file = tempfile.NamedTemporaryFile(suffix='.qgs', delete=False) + output_file.write(contents.encode('utf8')) + + QgsProject.instance().read(output_file.name) diff --git a/qgepplugin/ui/qgepdatamodeldialog.ui b/qgepplugin/ui/qgepdatamodeldialog.ui index ed4f2a7e..8fdcd591 100644 --- a/qgepplugin/ui/qgepdatamodeldialog.ui +++ b/qgepplugin/ui/qgepdatamodeldialog.ui @@ -7,48 +7,37 @@ 0 0 500 - 478 + 388 Datamodel tool - - + + + + + Target version + + + + + + + Depdencies - - - Release files - - - - - - - color: rgb(170, 0, 0); -font-weight: bold; - - - Unknown - - - Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse - - - - Python requirements - + color: rgb(170, 0, 0); @@ -62,7 +51,7 @@ font-weight: bold; - + Install dependencies @@ -72,7 +61,7 @@ font-weight: bold; - + Postgres service configuration @@ -130,7 +119,7 @@ font-weight: bold; - + Datamodel @@ -172,17 +161,28 @@ font-weight: bold; - + - Target version + SRID + + + 2056 + + + + + + + Action + + + + - - - @@ -205,24 +205,10 @@ font-weight: bold; - - - - SRID - - - - - - - 2056 - - - - + QGIS project @@ -245,7 +231,17 @@ font-weight: bold; - + + + + Qt::Horizontal + + + QDialogButtonBox::Close + + + + Qt::Vertical @@ -258,13 +254,24 @@ font-weight: bold; - - - - Qt::Horizontal + + + + Release files - - QDialogButtonBox::Close + + + + + + color: rgb(170, 0, 0); +font-weight: bold; + + + Unknown + + + Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse From bad2f1b69835ce0b5cb884e7e1b7fee13e76017d Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Wed, 28 Oct 2020 20:30:39 +0100 Subject: [PATCH 06/35] avancement --- qgepplugin/gui/qgepdatamodeldialog.py | 114 +++++++++++++------------- qgepplugin/ui/qgepdatamodeldialog.ui | 63 ++++++++------ 2 files changed, 99 insertions(+), 78 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index a571ee48..8ce22206 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -32,7 +32,7 @@ import subprocess import psycopg2 -from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QProgressDialog, QApplication +from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QProgressDialog, QApplication, QPushButton from qgis.PyQt.QtCore import QUrl, QFile, QIODevice from qgis.PyQt.QtNetwork import QNetworkRequest @@ -41,6 +41,35 @@ from ..utils import get_ui_class +AVAILABLE_RELEASES = { + 'master': '8.0', # too dangerous to expose here + '1.5.2': '8.0', + '1.5.1': '8.0', + '1.5.0': '8.0', + '1.4.0': '7.0', +} +TEMP_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', 'datamodel-init') + +# Path for pg_service.conf +if os.environ.get('PGSERVICEFILE'): + PG_CONFIG_PATH = os.environ.get('PGSERVICEFILE') +elif os.environ.get('PGSYSCONFDIR'): + PG_CONFIG_PATH = os.path.join(os.environ.get('PGSYSCONFDIR'), 'pg_service.conf') +else: + PG_CONFIG_PATH = ' ~/.pg_service.conf' + +MAIN_DATAMODEL_RELEASE = '1.5.2' +QGEP_RELEASE = '8.0' + +# Derived urls/paths, may require adaptations if release structure changes +DATAMODEL_URL_TEMPLATE = 'https://github.com/QGEP/datamodel/archive/{}.zip' +REQUIREMENTS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'requirements.txt') +DELTAS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'delta') +INIT_SCRIPT_URL_TEMPLATE = "https://github.com/QGEP/datamodel/releases/download/{}/qgep_v{}_structure_with_value_lists.sql" +QGEP_PROJECT_URL_TEMPLATE = 'https://github.com/QGEP/QGEP/releases/download/v{}/qgep.zip' +QGEP_PROJECT_PATH_TEMPLATE = os.path.join(TEMP_DIR, "project", 'qgep.qgs') + + def qgep_datamodel_error_catcher(func): """Display QGEPDatamodelError in error messages rather than normal exception dialog""" @@ -88,34 +117,6 @@ def conf_dict(self): class QgepDatamodelInitToolDialog(QDialog, get_ui_class('qgepdatamodeldialog.ui')): - AVAILABLE_VERSIONS = { - # 'master': '8.0', # too dangerous to expose here - '1.5.2': '8.0', - '1.5.1': '8.0', - '1.5.0': '8.0', - '1.4.0': '7.0', - } - TEMP_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', 'datamodel-init') - - # Path for pg_service.conf - if os.environ.get('PGSERVICEFILE'): - PG_CONFIG_PATH = os.environ.get('PGSERVICEFILE') - elif os.environ.get('PGSYSCONFDIR'): - PG_CONFIG_PATH = os.path.join(os.environ.get('PGSYSCONFDIR'), 'pg_service.conf') - else: - PG_CONFIG_PATH = ' ~/.pg_service.conf' - - MAIN_DATAMODEL_RELEASE = '1.5.2' - QGEP_RELEASE = '8.0' - - # Derived urls/paths, may require adaptations if release structure changes - DATAMODEL_URL_TEMPLATE = 'https://github.com/QGEP/datamodel/archive/{}.zip' - REQUIREMENTS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'requirements.txt') - DELTAS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'delta') - INIT_SCRIPT_URL_TEMPLATE = "https://github.com/QGEP/datamodel/releases/download/{}/qgep_v{}_structure_with_value_lists.sql" - QGEP_PROJECT_URL_TEMPLATE = 'https://github.com/QGEP/QGEP/releases/download/v{}/qgep.zip' - QGEP_PROJECT_PATH_TEMPLATE = os.path.join(TEMP_DIR, "project", 'qgep.qgs') - def __init__(self, parent=None): QDialog.__init__(self, parent) self.setupUi(self) @@ -123,18 +124,18 @@ def __init__(self, parent=None): self.progress_dialog = None # Populate the versions - self.targetVersionComboBox.clear() - self.targetVersionComboBox.addItem('- SELECT TARGET VERSION -') - self.targetVersionComboBox.model().item(0).setEnabled(False) - for version in sorted(list(self.AVAILABLE_VERSIONS.keys()), reverse=True): - self.targetVersionComboBox.addItem(version) + self.releaseVersionComboBox.clear() + self.releaseVersionComboBox.addItem('- SELECT RELEASE VERSION -') + self.releaseVersionComboBox.model().item(0).setEnabled(False) + for version in sorted(list(AVAILABLE_RELEASES.keys()), reverse=True): + self.releaseVersionComboBox.addItem(version) # Show the pgconfig path - self.pgservicePathLabel.setText(self.PG_CONFIG_PATH) + self.pgservicePathLabel.setText(PG_CONFIG_PATH) # Connect some signals - self.targetVersionComboBox.activated.connect(self.switch_datamodel) + self.releaseVersionComboBox.activated.connect(self.switch_datamodel) self.installDepsButton.pressed.connect(self.install_requirements) @@ -156,7 +157,7 @@ def __init__(self, parent=None): @property def version(self): - return self.targetVersionComboBox.currentText() + return self.releaseVersionComboBox.currentText() @property def conf(self): @@ -167,6 +168,9 @@ def conf(self): def _show_progress(self, message): if self.progress_dialog is None: self.progress_dialog = QProgressDialog("Starting...", "Cancel", 0, 0) + cancel_button = QPushButton("Cancel") + cancel_button.setEnabled(False) + self.progress_dialog.setCancelButton(cancel_button) self.progress_dialog.setLabelText(message) self.progress_dialog.show() QApplication.processEvents() @@ -201,11 +205,11 @@ def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see return result.stdout.decode() def _download(self, url, filename): - os.makedirs(self.TEMP_DIR, exist_ok=True) + os.makedirs(TEMP_DIR, exist_ok=True) network_manager = QgsNetworkAccessManager.instance() reply = network_manager.blockingGet(QNetworkRequest(QUrl(url))) - download_path = os.path.join(self.TEMP_DIR, filename) + download_path = os.path.join(TEMP_DIR, filename) download_file = QFile(download_path) download_file.open(QIODevice.WriteOnly) download_file.write(reply.content()) @@ -214,8 +218,8 @@ def _download(self, url, filename): def _read_pgservice(self): config = configparser.ConfigParser() - if os.path.exists(self.PG_CONFIG_PATH): - config.read(self.PG_CONFIG_PATH) + if os.path.exists(PG_CONFIG_PATH): + config.read(PG_CONFIG_PATH) return config def _get_pgservice_configs_names(self): @@ -237,11 +241,11 @@ def write(self, content): content = content.replace(" = ", "=", 1) self.output_file.write(content.encode('utf-8')) - config.write(EqualsSpaceRemover(open(self.PG_CONFIG_PATH, 'wb'))) + config.write(EqualsSpaceRemover(open(PG_CONFIG_PATH, 'wb'))) def _get_current_version(self): # Dirty parsing of pum info - deltas_dir = self.DELTAS_PATH_TEMPLATE.format(self.version) + deltas_dir = DELTAS_PATH_TEMPLATE.format(self.version) if not os.path.exists(deltas_dir): return None @@ -276,8 +280,8 @@ def enable_buttons_if_ready(self): # Datamodel def check_datamodel(self): - requirements_exists = os.path.exists(self.REQUIREMENTS_PATH_TEMPLATE.format(self.version)) - deltas_exists = os.path.exists(self.DELTAS_PATH_TEMPLATE.format(self.version)) + requirements_exists = os.path.exists(REQUIREMENTS_PATH_TEMPLATE.format(self.version)) + deltas_exists = os.path.exists(DELTAS_PATH_TEMPLATE.format(self.version)) check = requirements_exists and deltas_exists @@ -306,7 +310,7 @@ def switch_datamodel(self, _=None): # Unzip datamodel_zip = zipfile.ZipFile(datamodel_path) - datamodel_zip.extractall(self.TEMP_DIR) + datamodel_zip.extractall(TEMP_DIR) # Cleanup # os.unlink(datamodel_path) @@ -327,7 +331,7 @@ def check_requirements(self): if not self.check_datamodel(): missing.append(('unknown', 'no datamodel')) else: - requirements = pkg_resources.parse_requirements(open(self.REQUIREMENTS_PATH_TEMPLATE.format(self.version))) + requirements = pkg_resources.parse_requirements(open(REQUIREMENTS_PATH_TEMPLATE.format(self.version))) for requirement in requirements: try: pkg_resources.require(str(requirement)) @@ -368,7 +372,7 @@ def install_requirements(self): self._show_progress("Installing python dependencies with pip") # Install dependencies - requirements_file_path = self.REQUIREMENTS_PATH_TEMPLATE.format(self.version) + requirements_file_path = REQUIREMENTS_PATH_TEMPLATE.format(self.version) QgsMessageLog.logMessage(f"Installing python dependencies from {requirements_file_path}", "QGEP") self._run_cmd(f'pip install -r {requirements_file_path}', error_message='Could not install python dependencies') @@ -460,7 +464,7 @@ def check_version(self): @qgep_datamodel_error_catcher def upgrade_version(self): - version = self.targetVersionComboBox.currentText() + version = self.releaseVersionComboBox.currentText() pgservice = self.pgserviceComboBox.currentText() confirm = QMessageBox() @@ -495,7 +499,7 @@ def upgrade_version(self): # also currently SRID doesn't work try: self._show_progress("Downloading the structure script") - url = self.INIT_SCRIPT_URL_TEMPLATE.format(self.version, self.version) + url = INIT_SCRIPT_URL_TEMPLATE.format(self.version, self.version) sql_path = self._download(url, f"structure_with_value_lists-{self.version}-{srid}.sql") # Dirty hack to customize SRID in a dump @@ -538,7 +542,7 @@ def upgrade_version(self): raise QGEPDatamodelError(str(e)) self._show_progress("Running pum upgrade") - deltas_dir = self.DELTAS_PATH_TEMPLATE.format(self.version) + deltas_dir = DELTAS_PATH_TEMPLATE.format(self.version) return self._run_cmd( f'pum upgrade -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir} -u {self.version} -v int SRID {srid}', cwd=os.path.dirname(deltas_dir), @@ -560,13 +564,13 @@ def upgrade_version(self): def load_project(self): # Unzip QGEP - qgep_release = self.AVAILABLE_VERSIONS[self.version] - url = self.QGEP_PROJECT_URL_TEMPLATE.format(qgep_release) + qgep_release = AVAILABLE_RELEASES[self.version] + url = QGEP_PROJECT_URL_TEMPLATE.format(qgep_release) qgep_path = self._download(url, 'qgep.zip') qgep_zip = zipfile.ZipFile(qgep_path) - qgep_zip.extractall(self.TEMP_DIR) + qgep_zip.extractall(TEMP_DIR) - with open(self.QGEP_PROJECT_PATH_TEMPLATE, 'r') as original_project: + with open(QGEP_PROJECT_PATH_TEMPLATE, 'r') as original_project: contents = original_project.read() # replace the service name diff --git a/qgepplugin/ui/qgepdatamodeldialog.ui b/qgepplugin/ui/qgepdatamodeldialog.ui index 8fdcd591..da346327 100644 --- a/qgepplugin/ui/qgepdatamodeldialog.ui +++ b/qgepplugin/ui/qgepdatamodeldialog.ui @@ -17,12 +17,12 @@ - Target version + Release version - + @@ -30,31 +30,48 @@ Depdencies - - - - Python requirements - - - - - - color: rgb(170, 0, 0); + + + + + color: rgb(170, 0, 0); font-weight: bold; - - - Unknown - - - Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse - - + + + Unknown + + + Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse + + + + + + + Install + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + - - + + - Install dependencies + Python requirements From 0ee856fb4a8f2e8507f6ba559131638c09ad8831 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Wed, 28 Oct 2020 21:00:19 +0100 Subject: [PATCH 07/35] avancement --- qgepplugin/gui/qgepdatamodeldialog.py | 179 +++++++++++++++++--------- qgepplugin/ui/qgepdatamodeldialog.ui | 10 ++ 2 files changed, 127 insertions(+), 62 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 8ce22206..a350167c 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -142,7 +142,10 @@ def __init__(self, parent=None): self.pgserviceComboBox.activated.connect(self.select_pgconfig) self.pgserviceAddButton.pressed.connect(self.add_pgconfig) + self.targetVersionComboBox.activated.connect(self.check_version) self.versionUpgradeButton.pressed.connect(self.upgrade_version) + self.initializeButton.pressed.connect(self.initialize_version) + self.loadProjectButton.pressed.connect(self.load_project) # Initialize the checks @@ -159,6 +162,10 @@ def __init__(self, parent=None): def version(self): return self.releaseVersionComboBox.currentText() + @property + def target_version(self): + return self.targetVersionComboBox.currentText() + @property def conf(self): return self.pgserviceComboBox.currentText() @@ -273,8 +280,8 @@ def showEvent(self, event): super().showEvent(event) def enable_buttons_if_ready(self): - self.versionUpgradeButton.setEnabled(all(self.checks.values())) self.installDepsButton.setEnabled(self.checks['datamodel'] and not self.checks['requirements']) + self.versionUpgradeButton.setEnabled(all(self.checks.values())) self.loadProjectButton.setEnabled(self.checks['datamodel'] and self.checks['pgconfig']) # Datamodel @@ -422,14 +429,41 @@ def select_pgconfig(self, _=None): # Version - def check_version(self): + def check_version(self, _=None): check = False + # target version + + # (re)populate the combobox + prev = self.targetVersionComboBox.currentText() + self.targetVersionComboBox.clear() + available_versions = set() + deltas_dir = DELTAS_PATH_TEMPLATE.format(self.version) + if os.path.exists(deltas_dir): + for f in os.listdir(deltas_dir): + if f.startswith('delta_'): + available_versions.add(f.split('_')[1]) + for available_version in sorted(list(available_versions), reverse=True): + self.targetVersionComboBox.addItem(available_version) + self.targetVersionComboBox.setCurrentText(prev) # restore + + target_version = self.targetVersionComboBox.currentText() + + # current version + + self.initializeButton.setVisible(False) + self.targetVersionComboBox.setVisible(True) + self.versionUpgradeButton.setVisible(True) + pgservice = self.pgserviceComboBox.currentText() if not pgservice: self.versionCheckLabel.setText('service not selected') self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + elif not available_versions: + self.versionCheckLabel.setText('no delta in datamodel') + self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + else: try: @@ -444,11 +478,11 @@ def check_version(self): check = True self.versionCheckLabel.setText('not initialized') self.versionCheckLabel.setStyleSheet('color: rgb(170, 65, 0);\nfont-weight: bold;') - elif current_version <= self.version: + elif current_version <= target_version: check = True self.versionCheckLabel.setText(current_version) self.versionCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') - elif current_version > self.version: + elif current_version > target_version: check = False self.versionCheckLabel.setText(f"{current_version} (cannot downgrade)") self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') @@ -457,18 +491,20 @@ def check_version(self): self.versionCheckLabel.setText(f"{current_version} (invalid version)") self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.initializeButton.setVisible(current_version is None) + self.targetVersionComboBox.setVisible(not current_version is None) + self.versionUpgradeButton.setVisible(not current_version is None) + self.checks['current_version'] = check self.enable_buttons_if_ready() return check @qgep_datamodel_error_catcher - def upgrade_version(self): - version = self.releaseVersionComboBox.currentText() - pgservice = self.pgserviceComboBox.currentText() + def initialize_version(self): confirm = QMessageBox() - confirm.setText(f"You are about to update the datamodel on {pgservice} to version {version}. ") + confirm.setText(f"You are about to initialize the datamodel on {self.conf} to version {self.version}. ") confirm.setInformativeText( "Please confirm that you have a backup of your data as this operation can result in data loss." ) @@ -477,74 +513,93 @@ def upgrade_version(self): if confirm.exec_() == QMessageBox.Apply: - self._show_progress("Upgrading the datamodel") + self._show_progress("Initializing the datamodel") srid = self.sridLineEdit.text() - try: - current_version = self._get_current_version() - except QGEPDatamodelError: - # Can happend if PUM is not initialized, unfortunately we can't really - # determine if this is a connection error or if PUM is not initailized - # see https://github.com/opengisch/pum/issues/96 - current_version = None + # If we can't get current version, it's probably that the DB is not initialized + # (or maybe we can't connect, but we can't know easily with PUM) - if current_version is None: - # If we can't get current version, it's probably that the DB is not initialized - # (or maybe we can't connect, but we can't know easily with PUM) + self._show_progress("Initializing the datamodel") - self._show_progress("Initializing the datamodel") + # TODO : this should be done by PUM directly (see https://github.com/opengisch/pum/issues/94) + # also currently SRID doesn't work + try: + self._show_progress("Downloading the structure script") + url = INIT_SCRIPT_URL_TEMPLATE.format(self.version, self.version) + sql_path = self._download(url, f"structure_with_value_lists-{self.version}-{srid}.sql") + + # Dirty hack to customize SRID in a dump + if srid != '2056': + with open(sql_path, 'r') as file: + contents = file.read() + contents = contents.replace('2056', srid) + with open(sql_path, 'w') as file: + file.write(contents) - # TODO : this should be done by PUM directly (see https://github.com/opengisch/pum/issues/94) - # also currently SRID doesn't work try: - self._show_progress("Downloading the structure script") - url = INIT_SCRIPT_URL_TEMPLATE.format(self.version, self.version) - sql_path = self._download(url, f"structure_with_value_lists-{self.version}-{srid}.sql") - - # Dirty hack to customize SRID in a dump - if srid != '2056': - with open(sql_path, 'r') as file: - contents = file.read() - contents = contents.replace('2056', srid) - with open(sql_path, 'w') as file: - file.write(contents) - - try: - conn = psycopg2.connect(f"service={self.conf}") - except psycopg2.Error: - # It may be that the database doesn't exist yet - # in that case, we try to connect to the postgres database and to create it from there - self._show_progress("Creating the database") - dbname = self._read_pgservice()[self.conf]['dbname'] - self._run_cmd( - f'psql -c "CREATE DATABASE {dbname};" "service={self.conf} dbname=postgres"', - error_message='Errors when initializing the database.' - ) - conn = psycopg2.connect(f"service={self.conf}") - - self._show_progress("Running the initialization scripts") - cur = conn.cursor() - cur.execute('CREATE SCHEMA IF NOT EXISTS qgep_sys;') - cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;') - # we cannot use this, as it doesn't support COPY statements - # this means we'll run through psql without transaction :-/ - # cur.execute(open(sql_path, "r").read()) - conn.commit() - cur.close() - conn.close() + conn = psycopg2.connect(f"service={self.conf}") + except psycopg2.Error: + # It may be that the database doesn't exist yet + # in that case, we try to connect to the postgres database and to create it from there + self._show_progress("Creating the database") + dbname = self._read_pgservice()[self.conf]['dbname'] self._run_cmd( - f'psql -f {sql_path} "service={self.conf}"', + f'psql -c "CREATE DATABASE {dbname};" "service={self.conf} dbname=postgres"', error_message='Errors when initializing the database.' ) + conn = psycopg2.connect(f"service={self.conf}") + + self._show_progress("Running the initialization scripts") + cur = conn.cursor() + cur.execute('CREATE SCHEMA IF NOT EXISTS qgep_sys;') + cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;') + # we cannot use this, as it doesn't support COPY statements + # this means we'll run through psql without transaction :-/ + # cur.execute(open(sql_path, "r").read()) + conn.commit() + cur.close() + conn.close() + self._run_cmd( + f'psql -f {sql_path} "service={self.conf}"', + error_message='Errors when initializing the database.' + ) + + except psycopg2.Error as e: + raise QGEPDatamodelError(str(e)) + + self.check_version() + + self._done_progress() + + success = QMessageBox() + success.setText("Datamodel successfully initialized") + success.setIcon(QMessageBox.Information) + success.exec_() + + @qgep_datamodel_error_catcher + def upgrade_version(self): + + confirm = QMessageBox() + confirm.setText(f"You are about to update the datamodel on {self.conf} to version {self.target_version}. ") + confirm.setInformativeText( + "Please confirm that you have a backup of your data as this operation can result in data loss." + ) + confirm.setStandardButtons(QMessageBox.Apply | QMessageBox.Cancel) + confirm.setIcon(QMessageBox.Warning) + + if confirm.exec_() == QMessageBox.Apply: + + self._show_progress("Upgrading the datamodel") + + srid = self.sridLineEdit.text() - except psycopg2.Error as e: - raise QGEPDatamodelError(str(e)) + current_version = self._get_current_version() self._show_progress("Running pum upgrade") deltas_dir = DELTAS_PATH_TEMPLATE.format(self.version) return self._run_cmd( - f'pum upgrade -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir} -u {self.version} -v int SRID {srid}', + f'pum upgrade -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir} -u {self.target_version} -v int SRID {srid}', cwd=os.path.dirname(deltas_dir), error_message='Errors when upgrading the database.' ) @@ -555,7 +610,7 @@ def upgrade_version(self): success = QMessageBox() success.setText("Datamodel successfully upgraded") - success.setIcon(QMessageBox.Success) + success.setIcon(QMessageBox.Information) success.exec_() # Project diff --git a/qgepplugin/ui/qgepdatamodeldialog.ui b/qgepplugin/ui/qgepdatamodeldialog.ui index da346327..7d7d67a2 100644 --- a/qgepplugin/ui/qgepdatamodeldialog.ui +++ b/qgepplugin/ui/qgepdatamodeldialog.ui @@ -200,6 +200,16 @@ font-weight: bold; + + + + Initialize + + + + + + From 4a11838cfae9cac999e038fb5b8f8ddbcd36ee4f Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Wed, 28 Oct 2020 21:12:00 +0100 Subject: [PATCH 08/35] avancement --- qgepplugin/gui/qgepdatamodeldialog.py | 89 +++++++++++++++++++-------- qgepplugin/ui/qgepdatamodeldialog.ui | 42 ++++++++++--- 2 files changed, 100 insertions(+), 31 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index a350167c..0fcc98ff 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -40,13 +40,16 @@ from ..utils import get_ui_class - +# TODO : get latest dynamically ? AVAILABLE_RELEASES = { - 'master': '8.0', # too dangerous to expose here - '1.5.2': '8.0', - '1.5.1': '8.0', + 'master': 'https://github.com/QGEP/datamodel/archive/master.zip', # TODO : if we expose this here, we should put a big red warning and not take it default + '1.5.2': 'https://github.com/QGEP/datamodel/archive/1.5.2.zip', +} +# Allows to pick which QGIS project matches the version (will take the biggest <= match) +DATAMODEL_QGEP_VERSIONS = { '1.5.0': '8.0', '1.4.0': '7.0', + '0': '6.2', } TEMP_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', 'datamodel-init') @@ -88,14 +91,23 @@ class QGEPDatamodelError(Exception): class QgepPgserviceEditorDialog(QDialog, get_ui_class('qgeppgserviceeditordialog.ui')): - def __init__(self, existing_config_names, parent=None): - QDialog.__init__(self, parent) + def __init__(self, cur_name, cur_config, taken_names): + super().__init__() self.setupUi(self) - self.existing_config_names = existing_config_names + self.taken_names = taken_names self.nameLineEdit.textChanged.connect(self.check_name) + self.nameLineEdit.setText(cur_name) + self.pgconfigHostLineEdit.setText(cur_config.get("host", "")) + self.pgconfigPortLineEdit.setText(cur_config.get("port", "")) + self.pgconfigDbLineEdit.setText(cur_config.get("dbname", "")) + self.pgconfigUserLineEdit.setText(cur_config.get("user", "")) + self.pgconfigPasswordLineEdit.setText(cur_config.get("password", "")) + + self.check_name(cur_name) + def check_name(self, new_text): - if new_text in self.existing_config_names: + if new_text in self.taken_names: self.nameCheckLabel.setText('will overwrite') self.nameCheckLabel.setStyleSheet('color: rgb(170, 65, 0);\nfont-weight: bold;') else: @@ -154,6 +166,7 @@ def __init__(self, parent=None): 'requirements': False, 'pgconfig': False, 'current_version': False, + 'project': False, } # Properties @@ -229,10 +242,6 @@ def _read_pgservice(self): config.read(PG_CONFIG_PATH) return config - def _get_pgservice_configs_names(self): - config = self._read_pgservice() - return config.sections() - def _write_pgservice_conf(self, service_name, config_dict): config = self._read_pgservice() config[service_name] = config_dict @@ -277,12 +286,13 @@ def showEvent(self, event): self.check_requirements() self.check_pgconfig() self.check_version() + self.check_project() super().showEvent(event) def enable_buttons_if_ready(self): self.installDepsButton.setEnabled(self.checks['datamodel'] and not self.checks['requirements']) self.versionUpgradeButton.setEnabled(all(self.checks.values())) - self.loadProjectButton.setEnabled(self.checks['datamodel'] and self.checks['pgconfig']) + self.loadProjectButton.setEnabled(self.checks['project']) # Datamodel @@ -313,7 +323,7 @@ def switch_datamodel(self, _=None): self._show_progress("Downloading the release") # Download files - datamodel_path = self._download(self.DATAMODEL_URL_TEMPLATE.format(self.version), 'datamodel.zip') + datamodel_path = self._download(AVAILABLE_RELEASES[self.version], 'datamodel.zip') # Unzip datamodel_zip = zipfile.ZipFile(datamodel_path) @@ -406,8 +416,9 @@ def check_pgconfig(self): return check def add_pgconfig(self): - existing_config_names = self._get_pgservice_configs_names() - add_dialog = QgepPgserviceEditorDialog(existing_config_names) + taken_names = self._read_pgservice().sections() + cur_config = self._read_pgservice()[self.conf] + add_dialog = QgepPgserviceEditorDialog(self.conf, cur_config, taken_names) if add_dialog.exec_() == QDialog.Accepted: name = add_dialog.conf_name() conf = add_dialog.conf_dict() @@ -418,7 +429,7 @@ def add_pgconfig(self): def update_pgconfig_combobox(self): self.pgserviceComboBox.clear() - config_names = self._get_pgservice_configs_names() + config_names = self._read_pgservice().sections() for config_name in config_names: self.pgserviceComboBox.addItem(config_name) self.pgserviceComboBox.setCurrentIndex(0) @@ -426,6 +437,7 @@ def update_pgconfig_combobox(self): def select_pgconfig(self, _=None): self.check_pgconfig() self.check_version() + self.check_project() # Version @@ -492,8 +504,8 @@ def check_version(self, _=None): self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') self.initializeButton.setVisible(current_version is None) - self.targetVersionComboBox.setVisible(not current_version is None) - self.versionUpgradeButton.setVisible(not current_version is None) + self.targetVersionComboBox.setVisible(current_version is not None) + self.versionUpgradeButton.setVisible(current_version is not None) self.checks['current_version'] = check self.enable_buttons_if_ready() @@ -569,6 +581,7 @@ def initialize_version(self): raise QGEPDatamodelError(str(e)) self.check_version() + self.check_project() self._done_progress() @@ -594,8 +607,6 @@ def upgrade_version(self): srid = self.sridLineEdit.text() - current_version = self._get_current_version() - self._show_progress("Running pum upgrade") deltas_dir = DELTAS_PATH_TEMPLATE.format(self.version) return self._run_cmd( @@ -615,12 +626,42 @@ def upgrade_version(self): # Project + @qgep_datamodel_error_catcher + def check_project(self): + + try: + current_version = self._get_current_version() + except QGEPDatamodelError: + # Can happend if PUM is not initialized, unfortunately we can't really + # determine if this is a connection error or if PUM is not initailized + # see https://github.com/opengisch/pum/issues/96 + current_version = None + + check = current_version is not None + + if check: + self.projectCheckLabel.setText('ok') + self.projectCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + else: + self.projectCheckLabel.setText('version not found') + self.projectCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + + self.checks['project'] = check + self.enable_buttons_if_ready() + + return check + @qgep_datamodel_error_catcher def load_project(self): - # Unzip QGEP - qgep_release = AVAILABLE_RELEASES[self.version] - url = QGEP_PROJECT_URL_TEMPLATE.format(qgep_release) + current_version = self._get_current_version() + + qgis_vers = None + for dm_vers in sorted(DATAMODEL_QGEP_VERSIONS): + if dm_vers <= current_version: + qgis_vers = DATAMODEL_QGEP_VERSIONS[dm_vers] + + url = QGEP_PROJECT_URL_TEMPLATE.format(qgis_vers) qgep_path = self._download(url, 'qgep.zip') qgep_zip = zipfile.ZipFile(qgep_path) qgep_zip.extractall(TEMP_DIR) diff --git a/qgepplugin/ui/qgepdatamodeldialog.ui b/qgepplugin/ui/qgepdatamodeldialog.ui index 7d7d67a2..04c80272 100644 --- a/qgepplugin/ui/qgepdatamodeldialog.ui +++ b/qgepplugin/ui/qgepdatamodeldialog.ui @@ -241,6 +241,41 @@ font-weight: bold; QGIS project + + + + + + color: rgb(170, 0, 0); +font-weight: bold; + + + Unknown + + + + + + + Load + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + @@ -248,13 +283,6 @@ font-weight: bold; - - - - Load - - - From 3c5bd40b023af6003c823b65708be4e7f73d2f8d Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 29 Oct 2020 09:53:32 +0100 Subject: [PATCH 09/35] allow to not save user/password in pgservice --- qgepplugin/gui/qgepdatamodeldialog.py | 20 ++++++++++++++--- qgepplugin/ui/qgeppgserviceeditordialog.ui | 26 ++++++++++++++++++++-- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 0fcc98ff..bd9d9ef2 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -96,6 +96,8 @@ def __init__(self, cur_name, cur_config, taken_names): self.setupUi(self) self.taken_names = taken_names self.nameLineEdit.textChanged.connect(self.check_name) + self.pgconfigUserCheckBox.toggled.connect(self.pgconfigUserLineEdit.setEnabled) + self.pgconfigPasswordCheckBox.toggled.connect(self.pgconfigPasswordLineEdit.setEnabled) self.nameLineEdit.setText(cur_name) self.pgconfigHostLineEdit.setText(cur_config.get("host", "")) @@ -104,6 +106,11 @@ def __init__(self, cur_name, cur_config, taken_names): self.pgconfigUserLineEdit.setText(cur_config.get("user", "")) self.pgconfigPasswordLineEdit.setText(cur_config.get("password", "")) + self.pgconfigUserCheckBox.setChecked(cur_config.get("user") is not None) + self.pgconfigPasswordCheckBox.setChecked(cur_config.get("password") is not None) + self.pgconfigUserLineEdit.setEnabled(cur_config.get("user") is not None) + self.pgconfigPasswordLineEdit.setEnabled(cur_config.get("password") is not None) + self.check_name(cur_name) def check_name(self, new_text): @@ -118,13 +125,20 @@ def conf_name(self): return self.nameLineEdit.text() def conf_dict(self): - return { + retval = { "host": self.pgconfigHostLineEdit.text(), "port": self.pgconfigPortLineEdit.text(), "dbname": self.pgconfigDbLineEdit.text(), - "user": self.pgconfigUserLineEdit.text(), - "password": self.pgconfigPasswordLineEdit.text(), } + if self.pgconfigUserCheckBox.isChecked(): + retval.update({ + "user": self.pgconfigUserLineEdit.text(), + }) + if self.pgconfigPasswordCheckBox.isChecked(): + retval.update({ + "password": self.pgconfigPasswordLineEdit.text(), + }) + return retval class QgepDatamodelInitToolDialog(QDialog, get_ui_class('qgepdatamodeldialog.ui')): diff --git a/qgepplugin/ui/qgeppgserviceeditordialog.ui b/qgepplugin/ui/qgeppgserviceeditordialog.ui index b235d574..45945ebc 100644 --- a/qgepplugin/ui/qgeppgserviceeditordialog.ui +++ b/qgepplugin/ui/qgeppgserviceeditordialog.ui @@ -77,7 +77,18 @@ font-weight: bold; - + + + + + + + + + + + + @@ -87,7 +98,18 @@ font-weight: bold; - + + + + + + + + + + + + From 232763c1d0d18e44d072a863f9e0b1c41b55808f Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 5 Nov 2020 14:51:51 +0100 Subject: [PATCH 10/35] --user flag for pip --- qgepplugin/gui/qgepdatamodeldialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index bd9d9ef2..2f98e9ec 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -405,7 +405,7 @@ def install_requirements(self): # Install dependencies requirements_file_path = REQUIREMENTS_PATH_TEMPLATE.format(self.version) QgsMessageLog.logMessage(f"Installing python dependencies from {requirements_file_path}", "QGEP") - self._run_cmd(f'pip install -r {requirements_file_path}', error_message='Could not install python dependencies') + self._run_cmd(f'pip install --user -r {requirements_file_path}', error_message='Could not install python dependencies') self._done_progress() From 857d830f9379e12d4cc56cc95374d59be566230f Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 6 Nov 2020 12:49:06 +0100 Subject: [PATCH 11/35] better output in case of error when install deps --- qgepplugin/gui/qgepdatamodeldialog.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 2f98e9ec..db12f37f 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -235,7 +235,12 @@ def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see if result.stderr: QgsMessageLog.logMessage(result.stderr.decode(), "QGEP", level=Qgis.Critical) if result.returncode: - raise QGEPDatamodelError(f"{error_message}\n{result.stdout.decode()}\n{result.stderr.decode()}") + message = f"{error_message}\nCommand :\n{shell_command}" + if result.stdout: + message += f"\n\nOutput :\n{result.stdout.decode()}" + if result.stderr: + message += f"\n\nError :\n{result.stderr.decode()}" + raise QGEPDatamodelError(message) return result.stdout.decode() def _download(self, url, filename): @@ -405,7 +410,9 @@ def install_requirements(self): # Install dependencies requirements_file_path = REQUIREMENTS_PATH_TEMPLATE.format(self.version) QgsMessageLog.logMessage(f"Installing python dependencies from {requirements_file_path}", "QGEP") - self._run_cmd(f'pip install --user -r {requirements_file_path}', error_message='Could not install python dependencies') + dependencies = " ".join([f"'{l.strip()}'" for l in open(requirements_file_path, 'r').read().splitlines() if l.strip()]) + command_line = 'the OSGeo4W shell' if os.name == 'nt' else 'the terminal' + self._run_cmd(f'pip install --user {dependencies}', error_message=f'Could not install python dependencies. You can try to run the command manually from {command_line}.') self._done_progress() From b1e9d7cff85fef6399db311f34f575efc323be7e Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Wed, 11 Nov 2020 12:06:20 +0100 Subject: [PATCH 12/35] fix encoding error on win7 --- qgepplugin/gui/qgepdatamodeldialog.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index db12f37f..c6fbf63c 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -24,6 +24,7 @@ # --------------------------------------------------------------------- import os +import sys import configparser import functools import zipfile @@ -231,17 +232,17 @@ def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see QgsMessageLog.logMessage(f"Running command : {shell_command}", "QGEP") result = subprocess.run(shell_command, cwd=cwd, shell=True, capture_output=True) if result.stdout: - QgsMessageLog.logMessage(result.stdout.decode(), "QGEP") + QgsMessageLog.logMessage(result.stdout.decode(sys.getdefaultencoding()), "QGEP") if result.stderr: - QgsMessageLog.logMessage(result.stderr.decode(), "QGEP", level=Qgis.Critical) + QgsMessageLog.logMessage(result.stderr.decode(sys.getdefaultencoding()), "QGEP", level=Qgis.Critical) if result.returncode: message = f"{error_message}\nCommand :\n{shell_command}" if result.stdout: - message += f"\n\nOutput :\n{result.stdout.decode()}" + message += f"\n\nOutput :\n{result.stdout.decode(sys.getdefaultencoding())}" if result.stderr: - message += f"\n\nError :\n{result.stderr.decode()}" + message += f"\n\nError :\n{result.stderr.decode(sys.getdefaultencoding())}" raise QGEPDatamodelError(message) - return result.stdout.decode() + return result.stdout.decode(sys.getdefaultencoding()) def _download(self, url, filename): os.makedirs(TEMP_DIR, exist_ok=True) From 6f8d4ce64d02d61e812f3303a9d64c8ef2b81294 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Wed, 11 Nov 2020 12:06:38 +0100 Subject: [PATCH 13/35] recognize 404 (or other) errors on download --- qgepplugin/gui/qgepdatamodeldialog.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index c6fbf63c..4c2d8915 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -35,7 +35,7 @@ from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QProgressDialog, QApplication, QPushButton from qgis.PyQt.QtCore import QUrl, QFile, QIODevice -from qgis.PyQt.QtNetwork import QNetworkRequest +from qgis.PyQt.QtNetwork import QNetworkRequest, QNetworkReply from qgis.core import QgsMessageLog, QgsNetworkAccessManager, Qgis, QgsProject @@ -244,12 +244,19 @@ def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see raise QGEPDatamodelError(message) return result.stdout.decode(sys.getdefaultencoding()) - def _download(self, url, filename): + def _download(self, url, filename, error_message=None): os.makedirs(TEMP_DIR, exist_ok=True) network_manager = QgsNetworkAccessManager.instance() reply = network_manager.blockingGet(QNetworkRequest(QUrl(url))) + if reply.error() != QNetworkReply.NoError: + if error_message: + error_message = f"{error_message}\n{reply.errorString()}" + else: + error_message = reply.errorString() + raise QGEPDatamodelError(error_message) download_path = os.path.join(TEMP_DIR, filename) + QgsMessageLog.logMessage(f"Downloading {url} to {download_path}", "QGEP") download_file = QFile(download_path) download_file.open(QIODevice.WriteOnly) download_file.write(reply.content()) @@ -411,7 +418,7 @@ def install_requirements(self): # Install dependencies requirements_file_path = REQUIREMENTS_PATH_TEMPLATE.format(self.version) QgsMessageLog.logMessage(f"Installing python dependencies from {requirements_file_path}", "QGEP") - dependencies = " ".join([f"'{l.strip()}'" for l in open(requirements_file_path, 'r').read().splitlines() if l.strip()]) + dependencies = " ".join([f'"{l.strip()}"' for l in open(requirements_file_path, 'r').read().splitlines() if l.strip()]) command_line = 'the OSGeo4W shell' if os.name == 'nt' else 'the terminal' self._run_cmd(f'pip install --user {dependencies}', error_message=f'Could not install python dependencies. You can try to run the command manually from {command_line}.') @@ -561,7 +568,7 @@ def initialize_version(self): try: self._show_progress("Downloading the structure script") url = INIT_SCRIPT_URL_TEMPLATE.format(self.version, self.version) - sql_path = self._download(url, f"structure_with_value_lists-{self.version}-{srid}.sql") + sql_path = self._download(url, f"structure_with_value_lists-{self.version}-{srid}.sql", error_message=f"Initialization release file not found for version {self.version}") # Dirty hack to customize SRID in a dump if srid != '2056': From c8d3e52279afc0e9a9c8e781af2474cde62f8abf Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 12 Nov 2020 14:47:13 +0100 Subject: [PATCH 14/35] use python -m pip instead of pip (seems command line executables aren't always usable on some system despite proper library installation) --- qgepplugin/gui/qgepdatamodeldialog.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 4c2d8915..e1ff9b6d 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -293,7 +293,7 @@ def _get_current_version(self): return None pum_info = self._run_cmd( - f'pum info -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir}', + f'python -m pum info -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir}', error_message='Could not get current version, are you sure the database is accessible ?' ) for line in pum_info.splitlines(): @@ -420,7 +420,7 @@ def install_requirements(self): QgsMessageLog.logMessage(f"Installing python dependencies from {requirements_file_path}", "QGEP") dependencies = " ".join([f'"{l.strip()}"' for l in open(requirements_file_path, 'r').read().splitlines() if l.strip()]) command_line = 'the OSGeo4W shell' if os.name == 'nt' else 'the terminal' - self._run_cmd(f'pip install --user {dependencies}', error_message=f'Could not install python dependencies. You can try to run the command manually from {command_line}.') + self._run_cmd(f'python -m pip install --user {dependencies}', error_message=f'Could not install python dependencies. You can try to run the command manually from {command_line}.') self._done_progress() @@ -639,7 +639,7 @@ def upgrade_version(self): self._show_progress("Running pum upgrade") deltas_dir = DELTAS_PATH_TEMPLATE.format(self.version) return self._run_cmd( - f'pum upgrade -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir} -u {self.target_version} -v int SRID {srid}', + f'python -m pum upgrade -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir} -u {self.target_version} -v int SRID {srid}', cwd=os.path.dirname(deltas_dir), error_message='Errors when upgrading the database.' ) From 454642fb223e2a5f2fe37275091e564c2db7f6d8 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 12 Nov 2020 14:48:19 +0100 Subject: [PATCH 15/35] version 1.5.2 -> version 1.5.3 --- qgepplugin/gui/qgepdatamodeldialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index e1ff9b6d..552dadac 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -44,7 +44,7 @@ # TODO : get latest dynamically ? AVAILABLE_RELEASES = { 'master': 'https://github.com/QGEP/datamodel/archive/master.zip', # TODO : if we expose this here, we should put a big red warning and not take it default - '1.5.2': 'https://github.com/QGEP/datamodel/archive/1.5.2.zip', + '1.5.3': 'https://github.com/QGEP/datamodel/archive/1.5.3.zip', } # Allows to pick which QGIS project matches the version (will take the biggest <= match) DATAMODEL_QGEP_VERSIONS = { From 3ea4ce39a6974300b40316a21f8703ca5eb44897 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 12 Nov 2020 16:16:46 +0100 Subject: [PATCH 16/35] fix init scripts URL --- qgepplugin/gui/qgepdatamodeldialog.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 552dadac..40e0d84b 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -48,9 +48,9 @@ } # Allows to pick which QGIS project matches the version (will take the biggest <= match) DATAMODEL_QGEP_VERSIONS = { - '1.5.0': '8.0', - '1.4.0': '7.0', - '0': '6.2', + '1.5.0': 'v8.0', + '1.4.0': 'v7.0', + '0': 'v6.2', } TEMP_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', 'datamodel-init') @@ -69,8 +69,8 @@ DATAMODEL_URL_TEMPLATE = 'https://github.com/QGEP/datamodel/archive/{}.zip' REQUIREMENTS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'requirements.txt') DELTAS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'delta') -INIT_SCRIPT_URL_TEMPLATE = "https://github.com/QGEP/datamodel/releases/download/{}/qgep_v{}_structure_with_value_lists.sql" -QGEP_PROJECT_URL_TEMPLATE = 'https://github.com/QGEP/QGEP/releases/download/v{}/qgep.zip' +INIT_SCRIPT_URL_TEMPLATE = "https://github.com/QGEP/datamodel/releases/download/{}/qgep_{}_structure_with_value_lists.sql" +QGEP_PROJECT_URL_TEMPLATE = 'https://github.com/QGEP/QGEP/releases/download/{}/qgep.zip' QGEP_PROJECT_PATH_TEMPLATE = os.path.join(TEMP_DIR, "project", 'qgep.qgs') From d73bb33ac89bce99ca2d60325d0412a01a7cc73b Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 12 Nov 2020 16:39:40 +0100 Subject: [PATCH 17/35] better feedback for conection --- qgepplugin/gui/qgepdatamodeldialog.py | 36 ++++++++++++++++++++------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 40e0d84b..2e5fcfb3 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -89,7 +89,6 @@ def wrapper(*args, **kwargs): class QGEPDatamodelError(Exception): pass - class QgepPgserviceEditorDialog(QDialog, get_ui_class('qgeppgserviceeditordialog.ui')): def __init__(self, cur_name, cur_config, taken_names): @@ -117,7 +116,7 @@ def __init__(self, cur_name, cur_config, taken_names): def check_name(self, new_text): if new_text in self.taken_names: self.nameCheckLabel.setText('will overwrite') - self.nameCheckLabel.setStyleSheet('color: rgb(170, 65, 0);\nfont-weight: bold;') + self.nameCheckLabel.setStyleSheet('color: rgb(170, 95, 0);\nfont-weight: bold;') else: self.nameCheckLabel.setText('will be created') self.nameCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') @@ -470,6 +469,7 @@ def select_pgconfig(self, _=None): # Version + @qgep_datamodel_error_catcher def check_version(self, _=None): check = False @@ -507,18 +507,36 @@ def check_version(self, _=None): else: + error = None + current_version = None + connection_works = True + try: current_version = self._get_current_version() except QGEPDatamodelError: # Can happend if PUM is not initialized, unfortunately we can't really # determine if this is a connection error or if PUM is not initailized # see https://github.com/opengisch/pum/issues/96 - current_version = None - - if current_version is None: - check = True - self.versionCheckLabel.setText('not initialized') - self.versionCheckLabel.setStyleSheet('color: rgb(170, 65, 0);\nfont-weight: bold;') + # We'll try to connect to see if it's a connection error + error = 'qgep not initialized' + try: + self._run_cmd(f'psql -c "SELECT 1;" "service={self.conf}"') + except QGEPDatamodelError: + error = 'database does not exist' + try: + self._run_cmd(f'psql -c "SELECT 1;" "service={self.conf} dbname=postgres"') + except QGEPDatamodelError: + error = 'could not connect to database' + connection_works = False + + if not connection_works: + check = False + self.versionCheckLabel.setText(error) + self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + elif error is not None: + check = False + self.versionCheckLabel.setText(error) + self.versionCheckLabel.setStyleSheet('color: rgb(170, 95, 0);\nfont-weight: bold;') elif current_version <= target_version: check = True self.versionCheckLabel.setText(current_version) @@ -532,7 +550,7 @@ def check_version(self, _=None): self.versionCheckLabel.setText(f"{current_version} (invalid version)") self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') - self.initializeButton.setVisible(current_version is None) + self.initializeButton.setVisible(current_version is None and connection_works) self.targetVersionComboBox.setVisible(current_version is not None) self.versionUpgradeButton.setVisible(current_version is not None) From 76f690afa7620da52114f4a2b2547cdc270d8130 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 12 Nov 2020 17:50:49 +0100 Subject: [PATCH 18/35] refresh network layers on initialization --- qgepplugin/gui/qgepdatamodeldialog.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 2e5fcfb3..9b35ab63 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -623,6 +623,11 @@ def initialize_version(self): f'psql -f {sql_path} "service={self.conf}"', error_message='Errors when initializing the database.' ) + # workaround until https://github.com/QGEP/QGEP/issues/612 is fixed + self._run_cmd( + f'psql -c "SELECT qgep_network.refresh_network_simple();" "service={self.conf}"', + error_message='Errors when initializing the database.' + ) except psycopg2.Error as e: raise QGEPDatamodelError(str(e)) From 6b3b298f2651e3ae319f67440db98aff6303c22a Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 13 Nov 2020 11:59:42 +0100 Subject: [PATCH 19/35] hide password in connection editor --- qgepplugin/ui/qgeppgserviceeditordialog.ui | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qgepplugin/ui/qgeppgserviceeditordialog.ui b/qgepplugin/ui/qgeppgserviceeditordialog.ui index 45945ebc..187e69fc 100644 --- a/qgepplugin/ui/qgeppgserviceeditordialog.ui +++ b/qgepplugin/ui/qgeppgserviceeditordialog.ui @@ -107,7 +107,11 @@ font-weight: bold; - + + + QLineEdit::PasswordEchoOnEdit + + From ccb255893af06aef3b76e687f1fae6188baede7c Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 13 Nov 2020 11:59:52 +0100 Subject: [PATCH 20/35] show connection details --- qgepplugin/gui/qgepdatamodeldialog.py | 23 ++++++++++++------ qgepplugin/ui/qgepdatamodeldialog.ui | 34 +++++++++++++++++++++------ 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 9b35ab63..231659f7 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -195,7 +195,7 @@ def target_version(self): @property def conf(self): - return self.pgserviceComboBox.currentText() + return self.pgserviceComboBox.currentData() # Feedback helpers @@ -430,7 +430,7 @@ def install_requirements(self): def check_pgconfig(self): - check = self.pgserviceComboBox.currentText() != '' + check = bool(self.pgserviceComboBox.currentData()) if check: self.pgconfigCheckLabel.setText('ok') self.pgconfigCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') @@ -452,17 +452,26 @@ def add_pgconfig(self): conf = add_dialog.conf_dict() self._write_pgservice_conf(name, conf) self.update_pgconfig_combobox() - self.pgserviceComboBox.setCurrentIndex(self.pgserviceComboBox.findText(name)) + self.pgserviceComboBox.setCurrentIndex(self.pgserviceComboBox.findData(name)) self.select_pgconfig() def update_pgconfig_combobox(self): self.pgserviceComboBox.clear() - config_names = self._read_pgservice().sections() - for config_name in config_names: - self.pgserviceComboBox.addItem(config_name) + for config_name in self._read_pgservice().sections(): + self.pgserviceComboBox.addItem(config_name, config_name) self.pgserviceComboBox.setCurrentIndex(0) def select_pgconfig(self, _=None): + config = self._read_pgservice() + if self.conf in config.sections(): + host = config.get(self.conf, 'host', fallback='-') + port = config.get(self.conf, 'port', fallback='-') + dbname = config.get(self.conf, 'dbname', fallback='-') + user = config.get(self.conf, 'user', fallback='-') + password = (len(config.get(self.conf, 'password', fallback='')) * '*') or '-' + self.pgserviceCurrentLabel.setText(f"host: {host}:{port}\ndbname: {dbname}\nuser: {user}\npassword: {password}") + else: + self.pgserviceCurrentLabel.setText('-') self.check_pgconfig() self.check_version() self.check_project() @@ -496,7 +505,7 @@ def check_version(self, _=None): self.targetVersionComboBox.setVisible(True) self.versionUpgradeButton.setVisible(True) - pgservice = self.pgserviceComboBox.currentText() + pgservice = self.pgserviceComboBox.currentData() if not pgservice: self.versionCheckLabel.setText('service not selected') self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') diff --git a/qgepplugin/ui/qgepdatamodeldialog.ui b/qgepplugin/ui/qgepdatamodeldialog.ui index 04c80272..dd1fc0be 100644 --- a/qgepplugin/ui/qgepdatamodeldialog.ui +++ b/qgepplugin/ui/qgepdatamodeldialog.ui @@ -84,24 +84,31 @@ font-weight: bold; Postgres service configuration - + File location - + font: italic; color: #888 - TextLabel + - - + + + + PG Config setup + + + + @@ -126,10 +133,23 @@ font-weight: bold; - - + + - PG Config setup + Current selection + + + + + + + font: italic; color: #888 + + + - + + + true From d4200d16d5f2ff106a9a196dc7cc267fdea253b8 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 13 Nov 2020 12:10:05 +0100 Subject: [PATCH 21/35] remove creation of qgep_sys in initialization as it's done by the script already --- qgepplugin/gui/qgepdatamodeldialog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 231659f7..4d7ed296 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -620,7 +620,6 @@ def initialize_version(self): self._show_progress("Running the initialization scripts") cur = conn.cursor() - cur.execute('CREATE SCHEMA IF NOT EXISTS qgep_sys;') cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;') # we cannot use this, as it doesn't support COPY statements # this means we'll run through psql without transaction :-/ From 4997d2c360e2249261d08906c81ef25a665657d3 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 13 Nov 2020 15:47:21 +0100 Subject: [PATCH 22/35] use psycopg2 instead of psql to test connection (avoids hanging on password prompt) --- qgepplugin/gui/qgepdatamodeldialog.py | 50 ++++++++++++++++++++------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 4d7ed296..4faa5f4e 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -224,6 +224,21 @@ def _show_error(self, message): # Actions helpers + def _run_sql(self, connection_string, sql_command, error_message='Psycopg error, see logs for more information'): + QgsMessageLog.logMessage(f"Running query against {connection_string}: {sql_command}", "QGEP") + try: + conn = psycopg2.connect(connection_string) + cur = conn.cursor() + cur.execute(sql_command) + results = cur.fetchall() + conn.commit() + cur.close() + conn.close() + except psycopg2.OperationalError as e: + message = f"{error_message}\nCommand :\n{sql_command}\n{e}" + raise QGEPDatamodelError(message) + return results + def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see logs for more information'): """ Helper to run commands through subprocess @@ -529,11 +544,19 @@ def check_version(self, _=None): # We'll try to connect to see if it's a connection error error = 'qgep not initialized' try: - self._run_cmd(f'psql -c "SELECT 1;" "service={self.conf}"') + self._run_sql( + f"service={self.conf}", + 'SELECT 1;', + error_message='Errors when initializing the database.' + ) except QGEPDatamodelError: error = 'database does not exist' try: - self._run_cmd(f'psql -c "SELECT 1;" "service={self.conf} dbname=postgres"') + self._run_sql( + f"service={self.conf} dbname=postgres", + 'SELECT 1;', + error_message='Errors when initializing the database.' + ) except QGEPDatamodelError: error = 'could not connect to database' connection_works = False @@ -612,28 +635,29 @@ def initialize_version(self): # in that case, we try to connect to the postgres database and to create it from there self._show_progress("Creating the database") dbname = self._read_pgservice()[self.conf]['dbname'] - self._run_cmd( - f'psql -c "CREATE DATABASE {dbname};" "service={self.conf} dbname=postgres"', - error_message='Errors when initializing the database.' + self._run_sql( + f"service={self.conf} dbname=postgres", + f'CREATE DATABASE {dbname};', + error_message='Could not create a new database.' ) - conn = psycopg2.connect(f"service={self.conf}") self._show_progress("Running the initialization scripts") - cur = conn.cursor() - cur.execute('CREATE EXTENSION IF NOT EXISTS postgis;') + self._run_sql( + f"service={self.conf}", + 'CREATE EXTENSION IF NOT EXISTS postgis;', + error_message='Errors when initializing the database.' + ) # we cannot use this, as it doesn't support COPY statements # this means we'll run through psql without transaction :-/ # cur.execute(open(sql_path, "r").read()) - conn.commit() - cur.close() - conn.close() self._run_cmd( f'psql -f {sql_path} "service={self.conf}"', error_message='Errors when initializing the database.' ) # workaround until https://github.com/QGEP/QGEP/issues/612 is fixed - self._run_cmd( - f'psql -c "SELECT qgep_network.refresh_network_simple();" "service={self.conf}"', + self._run_sql( + f"service={self.conf}", + 'SELECT qgep_network.refresh_network_simple();', error_message='Errors when initializing the database.' ) From 104367f0d25a0085ae7172eec83df9c9f09f9fe9 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 13 Nov 2020 15:48:27 +0100 Subject: [PATCH 23/35] (avoid exception if puminfo fails) --- qgepplugin/gui/qgepdatamodeldialog.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 4faa5f4e..e0a5fc77 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -310,6 +310,7 @@ def _get_current_version(self): f'python -m pum info -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir}', error_message='Could not get current version, are you sure the database is accessible ?' ) + version = None for line in pum_info.splitlines(): line = line.strip() if not line: From e526ff7328d0f7ba6d423cfb25e7a00f9d7c2868 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 13 Nov 2020 16:00:52 +0100 Subject: [PATCH 24/35] fix psycopg2 errors --- qgepplugin/gui/qgepdatamodeldialog.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index e0a5fc77..c5394632 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -224,20 +224,20 @@ def _show_error(self, message): # Actions helpers - def _run_sql(self, connection_string, sql_command, error_message='Psycopg error, see logs for more information'): + def _run_sql(self, connection_string, sql_command, autocommit=False, error_message='Psycopg error, see logs for more information'): QgsMessageLog.logMessage(f"Running query against {connection_string}: {sql_command}", "QGEP") try: conn = psycopg2.connect(connection_string) + if autocommit: + conn.autocommit = True cur = conn.cursor() cur.execute(sql_command) - results = cur.fetchall() conn.commit() cur.close() conn.close() except psycopg2.OperationalError as e: message = f"{error_message}\nCommand :\n{sql_command}\n{e}" raise QGEPDatamodelError(message) - return results def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see logs for more information'): """ @@ -638,7 +638,7 @@ def initialize_version(self): dbname = self._read_pgservice()[self.conf]['dbname'] self._run_sql( f"service={self.conf} dbname=postgres", - f'CREATE DATABASE {dbname};', + f'CREATE DATABASE {dbname};', autocommit=True, error_message='Could not create a new database.' ) From 524f481743cbe72ae4cb0415db87e2a37fcec175 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 19 Nov 2020 16:07:19 +0100 Subject: [PATCH 25/35] add a timeout to run_cmd --- qgepplugin/gui/qgepdatamodeldialog.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index c5394632..fd05f9d3 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -239,12 +239,12 @@ def _run_sql(self, connection_string, sql_command, autocommit=False, error_messa message = f"{error_message}\nCommand :\n{sql_command}\n{e}" raise QGEPDatamodelError(message) - def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see logs for more information'): + def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see logs for more information', timeout=10): """ Helper to run commands through subprocess """ QgsMessageLog.logMessage(f"Running command : {shell_command}", "QGEP") - result = subprocess.run(shell_command, cwd=cwd, shell=True, capture_output=True) + result = subprocess.run(shell_command, cwd=cwd, shell=True, capture_output=True, timeout=timeout) if result.stdout: QgsMessageLog.logMessage(result.stdout.decode(sys.getdefaultencoding()), "QGEP") if result.stderr: @@ -435,7 +435,11 @@ def install_requirements(self): QgsMessageLog.logMessage(f"Installing python dependencies from {requirements_file_path}", "QGEP") dependencies = " ".join([f'"{l.strip()}"' for l in open(requirements_file_path, 'r').read().splitlines() if l.strip()]) command_line = 'the OSGeo4W shell' if os.name == 'nt' else 'the terminal' - self._run_cmd(f'python -m pip install --user {dependencies}', error_message=f'Could not install python dependencies. You can try to run the command manually from {command_line}.') + self._run_cmd( + f'python -m pip install --user {dependencies}', + error_message=f'Could not install python dependencies. You can try to run the command manually from {command_line}.', + timeout=None, + ) self._done_progress() @@ -653,7 +657,8 @@ def initialize_version(self): # cur.execute(open(sql_path, "r").read()) self._run_cmd( f'psql -f {sql_path} "service={self.conf}"', - error_message='Errors when initializing the database.' + error_message='Errors when initializing the database.', + timeout=300, ) # workaround until https://github.com/QGEP/QGEP/issues/612 is fixed self._run_sql( @@ -697,7 +702,8 @@ def upgrade_version(self): return self._run_cmd( f'python -m pum upgrade -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir} -u {self.target_version} -v int SRID {srid}', cwd=os.path.dirname(deltas_dir), - error_message='Errors when upgrading the database.' + error_message='Errors when upgrading the database.', + timeout=300, ) self.check_version() From 566fa9c46f64d1a5efbf7d4161d0422970d83829 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 19 Nov 2020 16:57:50 +0100 Subject: [PATCH 26/35] autoselect release version if just one --- qgepplugin/gui/qgepdatamodeldialog.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index fd05f9d3..c8c093ac 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -43,7 +43,7 @@ # TODO : get latest dynamically ? AVAILABLE_RELEASES = { - 'master': 'https://github.com/QGEP/datamodel/archive/master.zip', # TODO : if we expose this here, we should put a big red warning and not take it default + # 'master': 'https://github.com/QGEP/datamodel/archive/master.zip', # TODO : if we expose this here, we should put a big red warning and not take it default '1.5.3': 'https://github.com/QGEP/datamodel/archive/1.5.3.zip', } # Allows to pick which QGIS project matches the version (will take the biggest <= match) @@ -151,8 +151,9 @@ def __init__(self, parent=None): # Populate the versions self.releaseVersionComboBox.clear() - self.releaseVersionComboBox.addItem('- SELECT RELEASE VERSION -') - self.releaseVersionComboBox.model().item(0).setEnabled(False) + if len(AVAILABLE_RELEASES) > 1: + self.releaseVersionComboBox.addItem('- SELECT RELEASE VERSION -') + self.releaseVersionComboBox.model().item(0).setEnabled(False) for version in sorted(list(AVAILABLE_RELEASES.keys()), reverse=True): self.releaseVersionComboBox.addItem(version) @@ -183,6 +184,9 @@ def __init__(self, parent=None): 'project': False, } + if len(AVAILABLE_RELEASES) == 1: + self.switch_datamodel() + # Properties @property From e22e6a205d34f0f909083a1c543b859d9abbecbb Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Thu, 19 Nov 2020 17:24:41 +0100 Subject: [PATCH 27/35] add warning message about pg_config being unencrypted --- qgepplugin/ui/qgeppgserviceeditordialog.ui | 55 +++++++++++++++------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/qgepplugin/ui/qgeppgserviceeditordialog.ui b/qgepplugin/ui/qgeppgserviceeditordialog.ui index 187e69fc..5bed8335 100644 --- a/qgepplugin/ui/qgeppgserviceeditordialog.ui +++ b/qgepplugin/ui/qgeppgserviceeditordialog.ui @@ -6,22 +6,22 @@ 0 0 - 435 - 204 + 291 + 238 PG Config editor - + Config name - + @@ -39,44 +39,44 @@ font-weight: bold; - + Host - + - + Port - + - + Database - + - + User - + @@ -90,14 +90,14 @@ font-weight: bold; - + Password - + @@ -115,7 +115,7 @@ font-weight: bold; - + Qt::Vertical @@ -128,7 +128,7 @@ font-weight: bold; - + Qt::Horizontal @@ -138,6 +138,29 @@ font-weight: bold; + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + WARNING ! These settings will be saved to the pg_service.conf in plain text ! Any person having access to this file will be able to access the credentials. + + + true + + + From 4a297368570b40127e72228dc5defba431d2237b Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 4 Jun 2021 15:20:11 +0200 Subject: [PATCH 28/35] fix unloading settings menu item --- qgepplugin/qgepplugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepplugin/qgepplugin.py b/qgepplugin/qgepplugin.py index 7603f19b..2dbdf4ec 100644 --- a/qgepplugin/qgepplugin.py +++ b/qgepplugin/qgepplugin.py @@ -281,6 +281,7 @@ def unload(self): self.toolbar.deleteLater() self.iface.removePluginMenu("&QGEP", self.profileAction) + self.iface.removePluginMenu("&QGEP", self.settingsAction) self.iface.removePluginMenu("&QGEP", self.aboutAction) QgsApplication.processingRegistry().removeProvider(self.processing_provider) From b571ae8754c73ad54b8689811667631c10d0db75 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 4 Jun 2021 15:20:48 +0200 Subject: [PATCH 29/35] added "Admin mode" in settings to toggle display of datamodel init tool --- qgepplugin/gui/qgepsettingsdialog.py | 4 ++++ qgepplugin/qgepplugin.py | 6 ++++-- qgepplugin/ui/qgepsettingsdialog.ui | 14 ++++++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/qgepplugin/gui/qgepsettingsdialog.py b/qgepplugin/gui/qgepsettingsdialog.py index 99285405..fe4835ec 100644 --- a/qgepplugin/gui/qgepsettingsdialog.py +++ b/qgepplugin/gui/qgepsettingsdialog.py @@ -55,6 +55,9 @@ def __init__(self, parent=None): develmode = self.settings.value("/QGEP/DeveloperMode", False, type=bool) self.mCbDevelMode.setChecked(develmode) + adminmode = self.settings.value("/QGEP/AdminMode", False, type=bool) + self.mCbAdminMode.setChecked(adminmode) + lyr_special_structures, _ = project.readEntry('QGEP', 'SpecialStructureLayer') lyr_graph_edges, _ = project.readEntry('QGEP', 'GraphEdgeLayer') lyr_graph_nodes, _ = project.readEntry('QGEP', 'GraphNodeLayer') @@ -103,6 +106,7 @@ def onAccept(self): self.settings.remove("/QGEP/SvgProfilePath") self.settings.setValue("/QGEP/DeveloperMode", self.mCbDevelMode.checkState()) + self.settings.setValue("/QGEP/AdminMode", self.mCbAdminMode.checkState()) # Logging if hasattr(qgeplogger, 'qgepFileHandler'): diff --git a/qgepplugin/qgepplugin.py b/qgepplugin/qgepplugin.py index 2dbdf4ec..dd7ccc62 100644 --- a/qgepplugin/qgepplugin.py +++ b/qgepplugin/qgepplugin.py @@ -36,7 +36,7 @@ from qgis.PyQt.QtGui import QIcon from qgis.utils import qgsfunction -from qgis.core import QgsApplication +from qgis.core import QgsApplication, QgsSettings from .tools.qgepmaptools import ( QgepProfileMapTool, @@ -224,9 +224,10 @@ def initGui(self): self.toolbar.addAction(self.connectNetworkElementsAction) self.iface.addPluginToMenu("&QGEP", self.profileAction) - self.iface.addPluginToMenu("&QGEP", self.datamodelInitToolAction) self.iface.addPluginToMenu("&QGEP", self.settingsAction) self.iface.addPluginToMenu("&QGEP", self.aboutAction) + if QSettings().value("/QGEP/AdminMode", False): + self.iface.addPluginToMenu("&QGEP", self.datamodelInitToolAction) self.iface.addToolBar(self.toolbar) @@ -283,6 +284,7 @@ def unload(self): self.iface.removePluginMenu("&QGEP", self.profileAction) self.iface.removePluginMenu("&QGEP", self.settingsAction) self.iface.removePluginMenu("&QGEP", self.aboutAction) + self.iface.removePluginMenu("&QGEP", self.datamodelInitToolAction) QgsApplication.processingRegistry().removeProvider(self.processing_provider) diff --git a/qgepplugin/ui/qgepsettingsdialog.ui b/qgepplugin/ui/qgepsettingsdialog.ui index 1b46bade..c42330ad 100644 --- a/qgepplugin/ui/qgepsettingsdialog.ui +++ b/qgepplugin/ui/qgepsettingsdialog.ui @@ -175,7 +175,17 @@ - + + + + Enables the datamodel tool. The plugin needs to be reloaded. + + + Admin mode + + + + Layout @@ -212,7 +222,7 @@ - + Logging From 544826c864005c0e623df8fed8c5f62db3403f26 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 4 Jun 2021 15:28:50 +0200 Subject: [PATCH 30/35] [datainittool] bump latest release --- qgepplugin/gui/qgepdatamodeldialog.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index c8c093ac..40030374 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -41,10 +41,18 @@ from ..utils import get_ui_class -# TODO : get latest dynamically ? +# Currently, the latest release is hard-coded in the plugin, meaning we need +# to publish a plugin update for each datamodel update. +# In the future, once plugin/datamodel versionning scheme clearly reflects +# compatibility, we could retrieve this dynamically, so datamodel bugfix +# releases don't require a plugin upgrade. +LATEST_RELEASE = "1.5.4" + +# Allow to choose which releases can be installed +# (not so useful... but may want allow picking master for pre-releases) AVAILABLE_RELEASES = { # 'master': 'https://github.com/QGEP/datamodel/archive/master.zip', # TODO : if we expose this here, we should put a big red warning and not take it default - '1.5.3': 'https://github.com/QGEP/datamodel/archive/1.5.3.zip', + LATEST_RELEASE: f'https://github.com/QGEP/datamodel/archive/{LATEST_RELEASE}.zip', } # Allows to pick which QGIS project matches the version (will take the biggest <= match) DATAMODEL_QGEP_VERSIONS = { @@ -156,6 +164,7 @@ def __init__(self, parent=None): self.releaseVersionComboBox.model().item(0).setEnabled(False) for version in sorted(list(AVAILABLE_RELEASES.keys()), reverse=True): self.releaseVersionComboBox.addItem(version) + self.releaseVersionComboBox.setEnabled(len(AVAILABLE_RELEASES) > 1) # Show the pgconfig path self.pgservicePathLabel.setText(PG_CONFIG_PATH) From c41f9281b6b533938af322f42450c44f4eba7088 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 4 Jun 2021 16:55:43 +0200 Subject: [PATCH 31/35] formatting --- qgepplugin/gui/qgepdatamodeldialog.py | 392 ++++++++++++-------- qgepplugin/ui/qgeppgserviceeditordialog.ui | 402 ++++++++++----------- 2 files changed, 450 insertions(+), 344 deletions(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 40030374..0e958668 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -23,28 +23,33 @@ # # --------------------------------------------------------------------- -import os -import sys import configparser import functools -import zipfile +import os +import subprocess +import sys import tempfile +import zipfile + import pkg_resources -import subprocess import psycopg2 - -from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QProgressDialog, QApplication, QPushButton -from qgis.PyQt.QtCore import QUrl, QFile, QIODevice -from qgis.PyQt.QtNetwork import QNetworkRequest, QNetworkReply - -from qgis.core import QgsMessageLog, QgsNetworkAccessManager, Qgis, QgsProject +from qgis.core import Qgis, QgsMessageLog, QgsNetworkAccessManager, QgsProject +from qgis.PyQt.QtCore import QFile, QIODevice, QUrl +from qgis.PyQt.QtNetwork import QNetworkReply, QNetworkRequest +from qgis.PyQt.QtWidgets import ( + QApplication, + QDialog, + QMessageBox, + QProgressDialog, + QPushButton, +) from ..utils import get_ui_class # Currently, the latest release is hard-coded in the plugin, meaning we need # to publish a plugin update for each datamodel update. # In the future, once plugin/datamodel versionning scheme clearly reflects -# compatibility, we could retrieve this dynamically, so datamodel bugfix +# compatibility, we could retrieve this dynamically, so datamodel bugfix # releases don't require a plugin upgrade. LATEST_RELEASE = "1.5.4" @@ -52,34 +57,34 @@ # (not so useful... but may want allow picking master for pre-releases) AVAILABLE_RELEASES = { # 'master': 'https://github.com/QGEP/datamodel/archive/master.zip', # TODO : if we expose this here, we should put a big red warning and not take it default - LATEST_RELEASE: f'https://github.com/QGEP/datamodel/archive/{LATEST_RELEASE}.zip', + LATEST_RELEASE: f"https://github.com/QGEP/datamodel/archive/{LATEST_RELEASE}.zip", } # Allows to pick which QGIS project matches the version (will take the biggest <= match) DATAMODEL_QGEP_VERSIONS = { - '1.5.0': 'v8.0', - '1.4.0': 'v7.0', - '0': 'v6.2', + "1.5.0": "v8.0", + "1.4.0": "v7.0", + "0": "v6.2", } -TEMP_DIR = os.path.join(tempfile.gettempdir(), 'QGEP', 'datamodel-init') +TEMP_DIR = os.path.join(tempfile.gettempdir(), "QGEP", "datamodel-init") # Path for pg_service.conf -if os.environ.get('PGSERVICEFILE'): - PG_CONFIG_PATH = os.environ.get('PGSERVICEFILE') -elif os.environ.get('PGSYSCONFDIR'): - PG_CONFIG_PATH = os.path.join(os.environ.get('PGSYSCONFDIR'), 'pg_service.conf') +if os.environ.get("PGSERVICEFILE"): + PG_CONFIG_PATH = os.environ.get("PGSERVICEFILE") +elif os.environ.get("PGSYSCONFDIR"): + PG_CONFIG_PATH = os.path.join(os.environ.get("PGSYSCONFDIR"), "pg_service.conf") else: - PG_CONFIG_PATH = ' ~/.pg_service.conf' + PG_CONFIG_PATH = " ~/.pg_service.conf" -MAIN_DATAMODEL_RELEASE = '1.5.2' -QGEP_RELEASE = '8.0' +MAIN_DATAMODEL_RELEASE = "1.5.2" +QGEP_RELEASE = "8.0" # Derived urls/paths, may require adaptations if release structure changes -DATAMODEL_URL_TEMPLATE = 'https://github.com/QGEP/datamodel/archive/{}.zip' -REQUIREMENTS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'requirements.txt') -DELTAS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", 'delta') +DATAMODEL_URL_TEMPLATE = "https://github.com/QGEP/datamodel/archive/{}.zip" +REQUIREMENTS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", "requirements.txt") +DELTAS_PATH_TEMPLATE = os.path.join(TEMP_DIR, "datamodel-{}", "delta") INIT_SCRIPT_URL_TEMPLATE = "https://github.com/QGEP/datamodel/releases/download/{}/qgep_{}_structure_with_value_lists.sql" -QGEP_PROJECT_URL_TEMPLATE = 'https://github.com/QGEP/QGEP/releases/download/{}/qgep.zip' -QGEP_PROJECT_PATH_TEMPLATE = os.path.join(TEMP_DIR, "project", 'qgep.qgs') +QGEP_PROJECT_URL_TEMPLATE = "https://github.com/QGEP/QGEP/releases/download/{}/qgep.zip" +QGEP_PROJECT_PATH_TEMPLATE = os.path.join(TEMP_DIR, "project", "qgep.qgs") def qgep_datamodel_error_catcher(func): @@ -91,21 +96,24 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) except QGEPDatamodelError as e: args[0]._show_error(str(e)) + return wrapper class QGEPDatamodelError(Exception): pass -class QgepPgserviceEditorDialog(QDialog, get_ui_class('qgeppgserviceeditordialog.ui')): +class QgepPgserviceEditorDialog(QDialog, get_ui_class("qgeppgserviceeditordialog.ui")): def __init__(self, cur_name, cur_config, taken_names): super().__init__() self.setupUi(self) self.taken_names = taken_names self.nameLineEdit.textChanged.connect(self.check_name) self.pgconfigUserCheckBox.toggled.connect(self.pgconfigUserLineEdit.setEnabled) - self.pgconfigPasswordCheckBox.toggled.connect(self.pgconfigPasswordLineEdit.setEnabled) + self.pgconfigPasswordCheckBox.toggled.connect( + self.pgconfigPasswordLineEdit.setEnabled + ) self.nameLineEdit.setText(cur_name) self.pgconfigHostLineEdit.setText(cur_config.get("host", "")) @@ -123,11 +131,15 @@ def __init__(self, cur_name, cur_config, taken_names): def check_name(self, new_text): if new_text in self.taken_names: - self.nameCheckLabel.setText('will overwrite') - self.nameCheckLabel.setStyleSheet('color: rgb(170, 95, 0);\nfont-weight: bold;') + self.nameCheckLabel.setText("will overwrite") + self.nameCheckLabel.setStyleSheet( + "color: rgb(170, 95, 0);\nfont-weight: bold;" + ) else: - self.nameCheckLabel.setText('will be created') - self.nameCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + self.nameCheckLabel.setText("will be created") + self.nameCheckLabel.setStyleSheet( + "color: rgb(0, 170, 0);\nfont-weight: bold;" + ) def conf_name(self): return self.nameLineEdit.text() @@ -139,18 +151,21 @@ def conf_dict(self): "dbname": self.pgconfigDbLineEdit.text(), } if self.pgconfigUserCheckBox.isChecked(): - retval.update({ - "user": self.pgconfigUserLineEdit.text(), - }) + retval.update( + { + "user": self.pgconfigUserLineEdit.text(), + } + ) if self.pgconfigPasswordCheckBox.isChecked(): - retval.update({ - "password": self.pgconfigPasswordLineEdit.text(), - }) + retval.update( + { + "password": self.pgconfigPasswordLineEdit.text(), + } + ) return retval -class QgepDatamodelInitToolDialog(QDialog, get_ui_class('qgepdatamodeldialog.ui')): - +class QgepDatamodelInitToolDialog(QDialog, get_ui_class("qgepdatamodeldialog.ui")): def __init__(self, parent=None): QDialog.__init__(self, parent) self.setupUi(self) @@ -160,7 +175,7 @@ def __init__(self, parent=None): # Populate the versions self.releaseVersionComboBox.clear() if len(AVAILABLE_RELEASES) > 1: - self.releaseVersionComboBox.addItem('- SELECT RELEASE VERSION -') + self.releaseVersionComboBox.addItem("- SELECT RELEASE VERSION -") self.releaseVersionComboBox.model().item(0).setEnabled(False) for version in sorted(list(AVAILABLE_RELEASES.keys()), reverse=True): self.releaseVersionComboBox.addItem(version) @@ -186,11 +201,11 @@ def __init__(self, parent=None): # Initialize the checks self.checks = { - 'datamodel': False, - 'requirements': False, - 'pgconfig': False, - 'current_version': False, - 'project': False, + "datamodel": False, + "requirements": False, + "pgconfig": False, + "current_version": False, + "project": False, } if len(AVAILABLE_RELEASES) == 1: @@ -237,8 +252,16 @@ def _show_error(self, message): # Actions helpers - def _run_sql(self, connection_string, sql_command, autocommit=False, error_message='Psycopg error, see logs for more information'): - QgsMessageLog.logMessage(f"Running query against {connection_string}: {sql_command}", "QGEP") + def _run_sql( + self, + connection_string, + sql_command, + autocommit=False, + error_message="Psycopg error, see logs for more information", + ): + QgsMessageLog.logMessage( + f"Running query against {connection_string}: {sql_command}", "QGEP" + ) try: conn = psycopg2.connect(connection_string) if autocommit: @@ -252,22 +275,40 @@ def _run_sql(self, connection_string, sql_command, autocommit=False, error_messa message = f"{error_message}\nCommand :\n{sql_command}\n{e}" raise QGEPDatamodelError(message) - def _run_cmd(self, shell_command, cwd=None, error_message='Subprocess error, see logs for more information', timeout=10): + def _run_cmd( + self, + shell_command, + cwd=None, + error_message="Subprocess error, see logs for more information", + timeout=10, + ): """ Helper to run commands through subprocess """ QgsMessageLog.logMessage(f"Running command : {shell_command}", "QGEP") - result = subprocess.run(shell_command, cwd=cwd, shell=True, capture_output=True, timeout=timeout) + result = subprocess.run( + shell_command, cwd=cwd, shell=True, capture_output=True, timeout=timeout + ) if result.stdout: - QgsMessageLog.logMessage(result.stdout.decode(sys.getdefaultencoding()), "QGEP") + QgsMessageLog.logMessage( + result.stdout.decode(sys.getdefaultencoding()), "QGEP" + ) if result.stderr: - QgsMessageLog.logMessage(result.stderr.decode(sys.getdefaultencoding()), "QGEP", level=Qgis.Critical) + QgsMessageLog.logMessage( + result.stderr.decode(sys.getdefaultencoding()), + "QGEP", + level=Qgis.Critical, + ) if result.returncode: message = f"{error_message}\nCommand :\n{shell_command}" if result.stdout: - message += f"\n\nOutput :\n{result.stdout.decode(sys.getdefaultencoding())}" + message += ( + f"\n\nOutput :\n{result.stdout.decode(sys.getdefaultencoding())}" + ) if result.stderr: - message += f"\n\nError :\n{result.stderr.decode(sys.getdefaultencoding())}" + message += ( + f"\n\nError :\n{result.stderr.decode(sys.getdefaultencoding())}" + ) raise QGEPDatamodelError(message) return result.stdout.decode(sys.getdefaultencoding()) @@ -309,9 +350,9 @@ def __init__(self, output_file): def write(self, content): content = content.replace(" = ", "=", 1) - self.output_file.write(content.encode('utf-8')) + self.output_file.write(content.encode("utf-8")) - config.write(EqualsSpaceRemover(open(PG_CONFIG_PATH, 'wb'))) + config.write(EqualsSpaceRemover(open(PG_CONFIG_PATH, "wb"))) def _get_current_version(self): # Dirty parsing of pum info @@ -320,15 +361,15 @@ def _get_current_version(self): return None pum_info = self._run_cmd( - f'python -m pum info -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir}', - error_message='Could not get current version, are you sure the database is accessible ?' + f"python -m pum info -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir}", + error_message="Could not get current version, are you sure the database is accessible ?", ) version = None for line in pum_info.splitlines(): line = line.strip() if not line: continue - parts = line.split('|') + parts = line.split("|") if len(parts) > 1: version = parts[1].strip() return version @@ -345,26 +386,34 @@ def showEvent(self, event): super().showEvent(event) def enable_buttons_if_ready(self): - self.installDepsButton.setEnabled(self.checks['datamodel'] and not self.checks['requirements']) + self.installDepsButton.setEnabled( + self.checks["datamodel"] and not self.checks["requirements"] + ) self.versionUpgradeButton.setEnabled(all(self.checks.values())) - self.loadProjectButton.setEnabled(self.checks['project']) + self.loadProjectButton.setEnabled(self.checks["project"]) # Datamodel def check_datamodel(self): - requirements_exists = os.path.exists(REQUIREMENTS_PATH_TEMPLATE.format(self.version)) + requirements_exists = os.path.exists( + REQUIREMENTS_PATH_TEMPLATE.format(self.version) + ) deltas_exists = os.path.exists(DELTAS_PATH_TEMPLATE.format(self.version)) check = requirements_exists and deltas_exists if check: - self.releaseCheckLabel.setText('ok') - self.releaseCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + self.releaseCheckLabel.setText("ok") + self.releaseCheckLabel.setStyleSheet( + "color: rgb(0, 170, 0);\nfont-weight: bold;" + ) else: - self.releaseCheckLabel.setText('not found') - self.releaseCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.releaseCheckLabel.setText("not found") + self.releaseCheckLabel.setStyleSheet( + "color: rgb(170, 0, 0);\nfont-weight: bold;" + ) - self.checks['datamodel'] = check + self.checks["datamodel"] = check self.enable_buttons_if_ready() return check @@ -378,7 +427,9 @@ def switch_datamodel(self, _=None): self._show_progress("Downloading the release") # Download files - datamodel_path = self._download(AVAILABLE_RELEASES[self.version], 'datamodel.zip') + datamodel_path = self._download( + AVAILABLE_RELEASES[self.version], "datamodel.zip" + ) # Unzip datamodel_zip = zipfile.ZipFile(datamodel_path) @@ -401,27 +452,35 @@ def check_requirements(self): missing = [] if not self.check_datamodel(): - missing.append(('unknown', 'no datamodel')) + missing.append(("unknown", "no datamodel")) else: - requirements = pkg_resources.parse_requirements(open(REQUIREMENTS_PATH_TEMPLATE.format(self.version))) + requirements = pkg_resources.parse_requirements( + open(REQUIREMENTS_PATH_TEMPLATE.format(self.version)) + ) for requirement in requirements: try: pkg_resources.require(str(requirement)) except pkg_resources.DistributionNotFound: - missing.append((requirement, 'missing')) + missing.append((requirement, "missing")) except pkg_resources.VersionConflict: - missing.append((requirement, 'conflict')) + missing.append((requirement, "conflict")) check = len(missing) == 0 if check: - self.pythonCheckLabel.setText('ok') - self.pythonCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + self.pythonCheckLabel.setText("ok") + self.pythonCheckLabel.setStyleSheet( + "color: rgb(0, 170, 0);\nfont-weight: bold;" + ) else: - self.pythonCheckLabel.setText('\n'.join(f'{dep}: {err}' for dep, err in missing)) - self.pythonCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.pythonCheckLabel.setText( + "\n".join(f"{dep}: {err}" for dep, err in missing) + ) + self.pythonCheckLabel.setStyleSheet( + "color: rgb(170, 0, 0);\nfont-weight: bold;" + ) - self.checks['requirements'] = check + self.checks["requirements"] = check self.enable_buttons_if_ready() return check @@ -445,12 +504,20 @@ def install_requirements(self): # Install dependencies requirements_file_path = REQUIREMENTS_PATH_TEMPLATE.format(self.version) - QgsMessageLog.logMessage(f"Installing python dependencies from {requirements_file_path}", "QGEP") - dependencies = " ".join([f'"{l.strip()}"' for l in open(requirements_file_path, 'r').read().splitlines() if l.strip()]) - command_line = 'the OSGeo4W shell' if os.name == 'nt' else 'the terminal' + QgsMessageLog.logMessage( + f"Installing python dependencies from {requirements_file_path}", "QGEP" + ) + dependencies = " ".join( + [ + f'"{l.strip()}"' + for l in open(requirements_file_path, "r").read().splitlines() + if l.strip() + ] + ) + command_line = "the OSGeo4W shell" if os.name == "nt" else "the terminal" self._run_cmd( - f'python -m pip install --user {dependencies}', - error_message=f'Could not install python dependencies. You can try to run the command manually from {command_line}.', + f"python -m pip install --user {dependencies}", + error_message=f"Could not install python dependencies. You can try to run the command manually from {command_line}.", timeout=None, ) @@ -465,13 +532,17 @@ def check_pgconfig(self): check = bool(self.pgserviceComboBox.currentData()) if check: - self.pgconfigCheckLabel.setText('ok') - self.pgconfigCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + self.pgconfigCheckLabel.setText("ok") + self.pgconfigCheckLabel.setStyleSheet( + "color: rgb(0, 170, 0);\nfont-weight: bold;" + ) else: - self.pgconfigCheckLabel.setText('not set') - self.pgconfigCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.pgconfigCheckLabel.setText("not set") + self.pgconfigCheckLabel.setStyleSheet( + "color: rgb(170, 0, 0);\nfont-weight: bold;" + ) - self.checks['pgconfig'] = check + self.checks["pgconfig"] = check self.enable_buttons_if_ready() return check @@ -485,7 +556,9 @@ def add_pgconfig(self): conf = add_dialog.conf_dict() self._write_pgservice_conf(name, conf) self.update_pgconfig_combobox() - self.pgserviceComboBox.setCurrentIndex(self.pgserviceComboBox.findData(name)) + self.pgserviceComboBox.setCurrentIndex( + self.pgserviceComboBox.findData(name) + ) self.select_pgconfig() def update_pgconfig_combobox(self): @@ -497,14 +570,18 @@ def update_pgconfig_combobox(self): def select_pgconfig(self, _=None): config = self._read_pgservice() if self.conf in config.sections(): - host = config.get(self.conf, 'host', fallback='-') - port = config.get(self.conf, 'port', fallback='-') - dbname = config.get(self.conf, 'dbname', fallback='-') - user = config.get(self.conf, 'user', fallback='-') - password = (len(config.get(self.conf, 'password', fallback='')) * '*') or '-' - self.pgserviceCurrentLabel.setText(f"host: {host}:{port}\ndbname: {dbname}\nuser: {user}\npassword: {password}") + host = config.get(self.conf, "host", fallback="-") + port = config.get(self.conf, "port", fallback="-") + dbname = config.get(self.conf, "dbname", fallback="-") + user = config.get(self.conf, "user", fallback="-") + password = ( + len(config.get(self.conf, "password", fallback="")) * "*" + ) or "-" + self.pgserviceCurrentLabel.setText( + f"host: {host}:{port}\ndbname: {dbname}\nuser: {user}\npassword: {password}" + ) else: - self.pgserviceCurrentLabel.setText('-') + self.pgserviceCurrentLabel.setText("-") self.check_pgconfig() self.check_version() self.check_project() @@ -524,8 +601,8 @@ def check_version(self, _=None): deltas_dir = DELTAS_PATH_TEMPLATE.format(self.version) if os.path.exists(deltas_dir): for f in os.listdir(deltas_dir): - if f.startswith('delta_'): - available_versions.add(f.split('_')[1]) + if f.startswith("delta_"): + available_versions.add(f.split("_")[1]) for available_version in sorted(list(available_versions), reverse=True): self.targetVersionComboBox.addItem(available_version) self.targetVersionComboBox.setCurrentText(prev) # restore @@ -540,12 +617,16 @@ def check_version(self, _=None): pgservice = self.pgserviceComboBox.currentData() if not pgservice: - self.versionCheckLabel.setText('service not selected') - self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.versionCheckLabel.setText("service not selected") + self.versionCheckLabel.setStyleSheet( + "color: rgb(170, 0, 0);\nfont-weight: bold;" + ) elif not available_versions: - self.versionCheckLabel.setText('no delta in datamodel') - self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.versionCheckLabel.setText("no delta in datamodel") + self.versionCheckLabel.setStyleSheet( + "color: rgb(170, 0, 0);\nfont-weight: bold;" + ) else: @@ -560,51 +641,63 @@ def check_version(self, _=None): # determine if this is a connection error or if PUM is not initailized # see https://github.com/opengisch/pum/issues/96 # We'll try to connect to see if it's a connection error - error = 'qgep not initialized' + error = "qgep not initialized" try: self._run_sql( f"service={self.conf}", - 'SELECT 1;', - error_message='Errors when initializing the database.' + "SELECT 1;", + error_message="Errors when initializing the database.", ) except QGEPDatamodelError: - error = 'database does not exist' + error = "database does not exist" try: self._run_sql( f"service={self.conf} dbname=postgres", - 'SELECT 1;', - error_message='Errors when initializing the database.' + "SELECT 1;", + error_message="Errors when initializing the database.", ) except QGEPDatamodelError: - error = 'could not connect to database' + error = "could not connect to database" connection_works = False if not connection_works: check = False self.versionCheckLabel.setText(error) - self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.versionCheckLabel.setStyleSheet( + "color: rgb(170, 0, 0);\nfont-weight: bold;" + ) elif error is not None: check = False self.versionCheckLabel.setText(error) - self.versionCheckLabel.setStyleSheet('color: rgb(170, 95, 0);\nfont-weight: bold;') + self.versionCheckLabel.setStyleSheet( + "color: rgb(170, 95, 0);\nfont-weight: bold;" + ) elif current_version <= target_version: check = True self.versionCheckLabel.setText(current_version) - self.versionCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + self.versionCheckLabel.setStyleSheet( + "color: rgb(0, 170, 0);\nfont-weight: bold;" + ) elif current_version > target_version: check = False self.versionCheckLabel.setText(f"{current_version} (cannot downgrade)") - self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.versionCheckLabel.setStyleSheet( + "color: rgb(170, 0, 0);\nfont-weight: bold;" + ) else: check = False self.versionCheckLabel.setText(f"{current_version} (invalid version)") - self.versionCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.versionCheckLabel.setStyleSheet( + "color: rgb(170, 0, 0);\nfont-weight: bold;" + ) - self.initializeButton.setVisible(current_version is None and connection_works) + self.initializeButton.setVisible( + current_version is None and connection_works + ) self.targetVersionComboBox.setVisible(current_version is not None) self.versionUpgradeButton.setVisible(current_version is not None) - self.checks['current_version'] = check + self.checks["current_version"] = check self.enable_buttons_if_ready() return check @@ -613,7 +706,9 @@ def check_version(self, _=None): def initialize_version(self): confirm = QMessageBox() - confirm.setText(f"You are about to initialize the datamodel on {self.conf} to version {self.version}. ") + confirm.setText( + f"You are about to initialize the datamodel on {self.conf} to version {self.version}. " + ) confirm.setInformativeText( "Please confirm that you have a backup of your data as this operation can result in data loss." ) @@ -636,14 +731,18 @@ def initialize_version(self): try: self._show_progress("Downloading the structure script") url = INIT_SCRIPT_URL_TEMPLATE.format(self.version, self.version) - sql_path = self._download(url, f"structure_with_value_lists-{self.version}-{srid}.sql", error_message=f"Initialization release file not found for version {self.version}") + sql_path = self._download( + url, + f"structure_with_value_lists-{self.version}-{srid}.sql", + error_message=f"Initialization release file not found for version {self.version}", + ) # Dirty hack to customize SRID in a dump - if srid != '2056': - with open(sql_path, 'r') as file: + if srid != "2056": + with open(sql_path, "r") as file: contents = file.read() - contents = contents.replace('2056', srid) - with open(sql_path, 'w') as file: + contents = contents.replace("2056", srid) + with open(sql_path, "w") as file: file.write(contents) try: @@ -652,32 +751,33 @@ def initialize_version(self): # It may be that the database doesn't exist yet # in that case, we try to connect to the postgres database and to create it from there self._show_progress("Creating the database") - dbname = self._read_pgservice()[self.conf]['dbname'] + dbname = self._read_pgservice()[self.conf]["dbname"] self._run_sql( f"service={self.conf} dbname=postgres", - f'CREATE DATABASE {dbname};', autocommit=True, - error_message='Could not create a new database.' + f"CREATE DATABASE {dbname};", + autocommit=True, + error_message="Could not create a new database.", ) self._show_progress("Running the initialization scripts") self._run_sql( f"service={self.conf}", - 'CREATE EXTENSION IF NOT EXISTS postgis;', - error_message='Errors when initializing the database.' + "CREATE EXTENSION IF NOT EXISTS postgis;", + error_message="Errors when initializing the database.", ) # we cannot use this, as it doesn't support COPY statements # this means we'll run through psql without transaction :-/ # cur.execute(open(sql_path, "r").read()) self._run_cmd( f'psql -f {sql_path} "service={self.conf}"', - error_message='Errors when initializing the database.', + error_message="Errors when initializing the database.", timeout=300, ) # workaround until https://github.com/QGEP/QGEP/issues/612 is fixed self._run_sql( f"service={self.conf}", - 'SELECT qgep_network.refresh_network_simple();', - error_message='Errors when initializing the database.' + "SELECT qgep_network.refresh_network_simple();", + error_message="Errors when initializing the database.", ) except psycopg2.Error as e: @@ -697,7 +797,9 @@ def initialize_version(self): def upgrade_version(self): confirm = QMessageBox() - confirm.setText(f"You are about to update the datamodel on {self.conf} to version {self.target_version}. ") + confirm.setText( + f"You are about to update the datamodel on {self.conf} to version {self.target_version}. " + ) confirm.setInformativeText( "Please confirm that you have a backup of your data as this operation can result in data loss." ) @@ -713,9 +815,9 @@ def upgrade_version(self): self._show_progress("Running pum upgrade") deltas_dir = DELTAS_PATH_TEMPLATE.format(self.version) return self._run_cmd( - f'python -m pum upgrade -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir} -u {self.target_version} -v int SRID {srid}', + f"python -m pum upgrade -p {self.conf} -t qgep_sys.pum_info -d {deltas_dir} -u {self.target_version} -v int SRID {srid}", cwd=os.path.dirname(deltas_dir), - error_message='Errors when upgrading the database.', + error_message="Errors when upgrading the database.", timeout=300, ) @@ -744,13 +846,17 @@ def check_project(self): check = current_version is not None if check: - self.projectCheckLabel.setText('ok') - self.projectCheckLabel.setStyleSheet('color: rgb(0, 170, 0);\nfont-weight: bold;') + self.projectCheckLabel.setText("ok") + self.projectCheckLabel.setStyleSheet( + "color: rgb(0, 170, 0);\nfont-weight: bold;" + ) else: - self.projectCheckLabel.setText('version not found') - self.projectCheckLabel.setStyleSheet('color: rgb(170, 0, 0);\nfont-weight: bold;') + self.projectCheckLabel.setText("version not found") + self.projectCheckLabel.setStyleSheet( + "color: rgb(170, 0, 0);\nfont-weight: bold;" + ) - self.checks['project'] = check + self.checks["project"] = check self.enable_buttons_if_ready() return check @@ -766,17 +872,17 @@ def load_project(self): qgis_vers = DATAMODEL_QGEP_VERSIONS[dm_vers] url = QGEP_PROJECT_URL_TEMPLATE.format(qgis_vers) - qgep_path = self._download(url, 'qgep.zip') + qgep_path = self._download(url, "qgep.zip") qgep_zip = zipfile.ZipFile(qgep_path) qgep_zip.extractall(TEMP_DIR) - with open(QGEP_PROJECT_PATH_TEMPLATE, 'r') as original_project: + with open(QGEP_PROJECT_PATH_TEMPLATE, "r") as original_project: contents = original_project.read() # replace the service name contents = contents.replace("service='pg_qgep'", f"service='{self.conf}'") - output_file = tempfile.NamedTemporaryFile(suffix='.qgs', delete=False) - output_file.write(contents.encode('utf8')) + output_file = tempfile.NamedTemporaryFile(suffix=".qgs", delete=False) + output_file.write(contents.encode("utf8")) QgsProject.instance().read(output_file.name) diff --git a/qgepplugin/ui/qgeppgserviceeditordialog.ui b/qgepplugin/ui/qgeppgserviceeditordialog.ui index 5bed8335..ef4ba6a1 100644 --- a/qgepplugin/ui/qgeppgserviceeditordialog.ui +++ b/qgepplugin/ui/qgeppgserviceeditordialog.ui @@ -1,201 +1,201 @@ - - - Dialog - - - - 0 - 0 - 291 - 238 - - - - PG Config editor - - - - - - Config name - - - - - - - - - - - - color: rgb(170, 0, 0); -font-weight: bold; - - - Unknown - - - - - - - - - Host - - - - - - - - - - Port - - - - - - - - - - Database - - - - - - - - - - User - - - - - - - - - - - - - - - - - - - - - Password - - - - - - - - - - - - - - - - QLineEdit::PasswordEchoOnEdit - - - - - - - - - Qt::Vertical - - - - 20 - 40 - - - - - - - - Qt::Horizontal - - - QDialogButtonBox::Cancel|QDialogButtonBox::Save - - - - - - - Qt::Vertical - - - - 20 - 40 - - - - - - - - WARNING ! These settings will be saved to the pg_service.conf in plain text ! Any person having access to this file will be able to access the credentials. - - - true - - - - - - - - - buttonBox - accepted() - Dialog - accept() - - - 248 - 254 - - - 157 - 274 - - - - - buttonBox - rejected() - Dialog - reject() - - - 316 - 260 - - - 286 - 274 - - - - - + + + Dialog + + + + 0 + 0 + 291 + 238 + + + + PG Config editor + + + + + + Config name + + + + + + + + + + + + color: rgb(170, 0, 0); +font-weight: bold; + + + Unknown + + + + + + + + + Host + + + + + + + + + + Port + + + + + + + + + + Database + + + + + + + + + + User + + + + + + + + + + + + + + + + + + + + + Password + + + + + + + + + + + + + + + + QLineEdit::PasswordEchoOnEdit + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Save + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + WARNING ! These settings will be saved to the pg_service.conf in plain text ! Any person having access to this file will be able to access the credentials. + + + true + + + + + + + + + buttonBox + accepted() + Dialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + Dialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + From e504dfe6eecaba1f71a902fb13b88da7be0303fc Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 4 Jun 2021 17:05:13 +0200 Subject: [PATCH 32/35] fix header --- qgepplugin/gui/qgepdatamodeldialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 0e958668..36b6b379 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -2,7 +2,7 @@ # ----------------------------------------------------------- # # Profile -# Copyright (C) 2012 Matthias Kuhn +# Copyright (C) 2021 Olivier Dalang # ----------------------------------------------------------- # # licensed under the terms of GNU GPL 2 From b898935a128818ddf02029ccaab3238d2ac5d3cb Mon Sep 17 00:00:00 2001 From: Olivier Dalang Date: Fri, 4 Jun 2021 16:56:28 +0200 Subject: [PATCH 33/35] PR review Co-authored-by: Matthias Kuhn --- qgepplugin/gui/qgepdatamodeldialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 36b6b379..eedda95e 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -229,7 +229,7 @@ def conf(self): def _show_progress(self, message): if self.progress_dialog is None: - self.progress_dialog = QProgressDialog("Starting...", "Cancel", 0, 0) + self.progress_dialog = QProgressDialog(self.tr("Starting..."), self.tr("Cancel"), 0, 0) cancel_button = QPushButton("Cancel") cancel_button.setEnabled(False) self.progress_dialog.setCancelButton(cancel_button) From f21b04deea9eb81be9990aecaf2fa93bf48d8bf3 Mon Sep 17 00:00:00 2001 From: Olivier Dalang Date: Fri, 4 Jun 2021 16:56:42 +0200 Subject: [PATCH 34/35] PR review Co-authored-by: Matthias Kuhn --- qgepplugin/gui/qgepdatamodeldialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index eedda95e..0546a719 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -230,7 +230,7 @@ def conf(self): def _show_progress(self, message): if self.progress_dialog is None: self.progress_dialog = QProgressDialog(self.tr("Starting..."), self.tr("Cancel"), 0, 0) - cancel_button = QPushButton("Cancel") + cancel_button = QPushButton(self.tr("Cancel")) cancel_button.setEnabled(False) self.progress_dialog.setCancelButton(cancel_button) self.progress_dialog.setLabelText(message) From 47097a983ea01fdfb56689019bb63eee4d318409 Mon Sep 17 00:00:00 2001 From: olivierdalang Date: Fri, 4 Jun 2021 17:12:12 +0200 Subject: [PATCH 35/35] (precommit) --- qgepplugin/gui/qgepdatamodeldialog.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qgepplugin/gui/qgepdatamodeldialog.py b/qgepplugin/gui/qgepdatamodeldialog.py index 0546a719..ca90fc29 100644 --- a/qgepplugin/gui/qgepdatamodeldialog.py +++ b/qgepplugin/gui/qgepdatamodeldialog.py @@ -229,7 +229,9 @@ def conf(self): def _show_progress(self, message): if self.progress_dialog is None: - self.progress_dialog = QProgressDialog(self.tr("Starting..."), self.tr("Cancel"), 0, 0) + self.progress_dialog = QProgressDialog( + self.tr("Starting..."), self.tr("Cancel"), 0, 0 + ) cancel_button = QPushButton(self.tr("Cancel")) cancel_button.setEnabled(False) self.progress_dialog.setCancelButton(cancel_button)