diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..265b122 --- /dev/null +++ b/.gitignore @@ -0,0 +1,118 @@ + +# Created by https://www.gitignore.io/api/code,python +# Edit at https://www.gitignore.io/?templates=code,python + +### Code ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# End of https://www.gitignore.io/api/code,python + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..912c1a3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +# v0.0.1 + +Pemière pre-release avec un premier lot de fonctions: +* Extraction de données à partir d'une couche de zonages +* Création d'un tableau de synthèse par taxons à partir d'une couche de zonages +* Création d'un graphique de synthèse d'état des connaissances par groupes taxonomiques à partir d'une couche de zonages + + diff --git a/README.md b/README.md new file mode 100644 index 0000000..99c7881 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# Scripts de processing framework de la LPO AuRA + +Ce plugin ajoute à QGIS des scripts d'exploitation des données naturalistes de +la [LPO Auvergne-Rhône-Alpes](https://auvergne-rhone-alpes.lpo.fr/) à QGIS. + +## Licence + +## Equipe + +* @eguilley (LPO Auvergne-Rhône-Alpes) +* @lpofredc (@lpofredc - LPO Auvergne-Rhône-Alpes) +* @jgirardclaudon (@lpojgc LPO Auvergne-Rhône-Alpes) + +![logoLPO AuRA](https://raw.githubusercontent.com/lpoaura/biodivsport-widget/master/images/LPO_AuRA_l250px.png) + + diff --git a/README.txt b/README.txt deleted file mode 100644 index 4de51bf..0000000 --- a/README.txt +++ /dev/null @@ -1,26 +0,0 @@ -Plugin Builder Results - -Your plugin ScriptsLPO was created in: - /home/eguilley/Dev/plugin_qgis/scripts_lpo/scripts_lpo - -Your QGIS plugin directory is located at: - /home/eguilley/.local/share/QGIS/QGIS3/profiles/default/python/plugins - -What's Next: - - * Copy the entire directory containing your new plugin to the QGIS plugin - directory - - * Run the tests (``make test``) - - * Test the plugin by enabling it in the QGIS plugin manager - - * Customize it by editing the implementation file: ``scripts_lpo.py`` - - * You can use the Makefile to compile your Ui and resource files when - you make changes. This requires GNU make (gmake) - -For more information, see the PyQGIS Developer Cookbook at: -http://www.qgis.org/pyqgis-cookbook/index.html - -(C) 2011-2018 GeoApt LLC - geoapt.com diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..95e94cd --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v0.0.1 \ No newline at end of file diff --git a/__pycache__/__init__.cpython-36.pyc b/__pycache__/__init__.cpython-36.pyc deleted file mode 100644 index 7250c3f..0000000 Binary files a/__pycache__/__init__.cpython-36.pyc and /dev/null differ diff --git a/__pycache__/extract_data.cpython-36.pyc b/__pycache__/extract_data.cpython-36.pyc deleted file mode 100644 index e052107..0000000 Binary files a/__pycache__/extract_data.cpython-36.pyc and /dev/null differ diff --git a/__pycache__/scripts_lpo.cpython-36.pyc b/__pycache__/scripts_lpo.cpython-36.pyc deleted file mode 100644 index c4fd5fd..0000000 Binary files a/__pycache__/scripts_lpo.cpython-36.pyc and /dev/null differ diff --git a/__pycache__/scripts_lpo_algorithm.cpython-36.pyc b/__pycache__/scripts_lpo_algorithm.cpython-36.pyc deleted file mode 100644 index c344da5..0000000 Binary files a/__pycache__/scripts_lpo_algorithm.cpython-36.pyc and /dev/null differ diff --git a/__pycache__/scripts_lpo_provider.cpython-36.pyc b/__pycache__/scripts_lpo_provider.cpython-36.pyc deleted file mode 100644 index e79edec..0000000 Binary files a/__pycache__/scripts_lpo_provider.cpython-36.pyc and /dev/null differ diff --git a/__pycache__/summary_table.cpython-36.pyc b/__pycache__/summary_table.cpython-36.pyc deleted file mode 100644 index f509320..0000000 Binary files a/__pycache__/summary_table.cpython-36.pyc and /dev/null differ diff --git a/common_functions.py b/common_functions.py new file mode 100644 index 0000000..b25c1df --- /dev/null +++ b/common_functions.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- + +""" +/*************************************************************************** + ScriptsLPO : common_functions.py + ------------------- + Date : 2020-04-16 + Copyright : (C) 2020 by Elsa Guilley (LPO AuRA) + Email : lpo-aura@lpo.fr + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +__author__ = 'Elsa Guilley (LPO AuRA)' +__date__ = '2020-04-16' +__copyright__ = '(C) 2020 by Elsa Guilley (LPO AuRA)' + +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +from qgis.utils import iface +from qgis.gui import QgsMessageBar + +from qgis.PyQt.QtCore import QVariant +from qgis.core import (QgsWkbTypes, + QgsField, + QgsProcessingException, + Qgis) +import processing + + +def simplify_name(string): + """ + Simplify a layer name written by the user. + """ + translation_table = str.maketrans( + 'àâäéèêëîïôöùûüŷÿç~- ', + 'aaaeeeeiioouuuyyc___', + "2&'([{|}])`^\/@+-=*°$£%§#.?!;:<>" + ) + return string.lower().translate(translation_table) + +# def check_layer_geometry(layer): +# """ +# Check if the input vector layer is a polygon layer. +# """ +# if QgsWkbTypes.displayString(layer.wkbType()) not in ['Polygon', 'MultiPolygon']: +# iface.messageBar().pushMessage("Erreur", "La zone d'étude fournie n'est pas valide ! Veuillez sélectionner une couche vecteur de type POLYGONE.", level=Qgis.Critical, duration=10) +# raise QgsProcessingException("La zone d'étude fournie n'est pas valide ! Veuillez sélectionner une couche vecteur de type POLYGONE.") +# return None + +def check_layer_is_valid(feedback, layer): + """ + Check if the input vector layer is valid. + """ + if not layer.isValid(): + raise QgsProcessingException(""""La couche PostGIS chargée n'est pas valide ! + Checkez les logs de PostGIS pour visualiser les messages d'erreur.""") + else: + #iface.messageBar().pushMessage("Info", "La couche PostGIS demandée est valide, la requête SQL a été exécutée avec succès !", level=Qgis.Info, duration=10) + feedback.pushInfo("La couche PostGIS demandée est valide, la requête SQL a été exécutée avec succès !") + return None + +def construct_sql_array_polygons(layer): + """ + Construct the sql array containing the input vector layer's features geometry. + """ + # Initialization of the sql array containing the study area's features geometry + array_polygons = "array[" + # Retrieve the CRS of the layer + crs = layer.sourceCrs().authid().split(':')[1] + # For each entity in the study area... + for feature in layer.getFeatures(): + # Retrieve the geometry + area = feature.geometry() # QgsGeometry object + # Retrieve the geometry type (single or multiple) + geomSingleType = QgsWkbTypes.isSingleType(area.wkbType()) + # Increment the sql array + if geomSingleType: + array_polygons += "ST_transform(ST_PolygonFromText('{}', {}), 2154), ".format(area.asWkt(), crs) + else: + array_polygons += "ST_transform(ST_MPolyFromText('{}', {}), 2154), ".format(area.asWkt(), crs) + # Remove the last "," in the sql array which is useless, and end the array + array_polygons = array_polygons[:len(array_polygons)-2] + "]" + return array_polygons + +def load_layer(context, layer): + """ + Load a layer in the current project. + """ + root = context.project().layerTreeRoot() + plugin_lpo_group = root.findGroup('Résultats plugin LPO') + if not plugin_lpo_group: + plugin_lpo_group = root.insertGroup(0, 'Résultats plugin LPO') + context.project().addMapLayer(layer, False) + plugin_lpo_group.addLayer(layer) + ### Variant + # context.temporaryLayerStore().addMapLayer(layer) + # context.addLayerToLoadOnCompletion( + # layer.id(), + # QgsProcessingContext.LayerDetails("Données d'observations", context.project(), self.OUTPUT) + # ) + +def execute_sql_queries(context, feedback, connection, queries): + """ + Execute several sql queries. + """ + for query in queries: + processing.run( + 'qgis:postgisexecutesql', + { + 'DATABASE': connection, + 'SQL': query + }, + is_child_algorithm=True, + context=context, + feedback=feedback + ) + feedback.pushInfo('Requête SQL exécutée avec succès !') + return None + +def format_layer_export(layer): + """ + Create new valid fields for the sink. + """ + old_fields = layer.fields() + new_fields = layer.fields() + new_fields.clear() + invalid_formats = ["_text", "jsonb"] + for field in old_fields: + if field.typeName() in invalid_formats: + new_fields.append(QgsField(field.name(), QVariant.String, "str")) + else: + new_fields.append(field) + # for i,field in enumerate(new_fields): + # feedback.pushInfo('Elt : {}- {} {}'.format(i, field.name(), field.typeName())) + return new_fields diff --git a/extract_data.py b/extract_data.py index bd00f79..452c21f 100644 --- a/extract_data.py +++ b/extract_data.py @@ -27,20 +27,24 @@ __revision__ = '$Format:%H$' import os +from datetime import datetime from qgis.PyQt.QtGui import QIcon -from qgis.PyQt.QtCore import QCoreApplication +from qgis.PyQt.QtCore import QCoreApplication, QVariant from qgis.core import (QgsProcessing, QgsProcessingAlgorithm, QgsProcessingParameterString, - QgsProcessingParameterVectorLayer, + QgsProcessingParameterFeatureSource, QgsProcessingOutputVectorLayer, + QgsProcessingParameterFeatureSink, QgsDataSourceUri, QgsVectorLayer, - QgsWkbTypes, - QgsProcessingContext, - QgsProcessingException) + QgsField, + QgsProcessingUtils, + QgsProcessingException, + QgsProject) from processing.tools import postgis +from .common_functions import check_layer_is_valid, construct_sql_array_polygons, load_layer, format_layer_export pluginPath = os.path.dirname(__file__) @@ -53,23 +57,25 @@ class ExtractData(QgsProcessingAlgorithm): # Constants used to refer to parameters and outputs DATABASE = 'DATABASE' - ZONE_ETUDE = 'ZONE_ETUDE' + STUDY_AREA = 'STUDY_AREA' OUTPUT = 'OUTPUT' + OUTPUT_NAME = 'OUTPUT_NAME' + dest_id = None def name(self): return 'ExtractData' def displayName(self): - return 'Extract observation data from study area' + return "Extraction de données d'observation" def icon(self): return QIcon(os.path.join(pluginPath, 'icons', 'extract_data.png')) def groupId(self): - return 'initialisation' + return 'test' def group(self): - return 'Initialisation' + return 'Test' def initAlgorithm(self, config=None): """ @@ -80,7 +86,7 @@ def initAlgorithm(self, config=None): # Data base connection db_param = QgsProcessingParameterString( self.DATABASE, - self.tr('Nom de la connexion à la base de données') + self.tr("1/ Sélectionnez votre connexion à la base de données LPO AuRA") ) db_param.setMetadata( { @@ -91,19 +97,31 @@ def initAlgorithm(self, config=None): # Input vector layer = study area self.addParameter( - QgsProcessingParameterVectorLayer( - self.ZONE_ETUDE, - self.tr("Zone d'étude"), - [QgsProcessing.TypeVectorAnyGeometry] + QgsProcessingParameterFeatureSource( + self.STUDY_AREA, + self.tr("2/ Sélectionnez votre zone d'étude, à partir de laquelle seront extraites les données d'observations"), + [QgsProcessing.TypeVectorPolygon] ) ) - # Output PostGIS layer - self.addOutput( - QgsProcessingOutputVectorLayer( + # Output PostGIS layer name + self.addParameter( + QgsProcessingParameterString( + self.OUTPUT_NAME, + self.tr("3/ Définissez un nom pour votre nouvelle couche"), + self.tr("Données d'observation") + ) + ) + + # Output PostGIS layer = biodiversity data + self.addParameter( + QgsProcessingParameterFeatureSink( self.OUTPUT, - self.tr('Couche en sortie'), - QgsProcessing.TypeVectorAnyGeometry + self.tr('4/ Si nécessaire, enregistrez votre nouvelle couche (vous pouvez aussi ignorer cette étape)'), + QgsProcessing.TypeVectorPoint, + None, + True, + False ) ) @@ -113,54 +131,46 @@ def processAlgorithm(self, parameters, context, feedback): """ # Retrieve the input vector layer = study area - zone_etude = self.parameterAsVectorLayer(parameters, self.ZONE_ETUDE, context) - # Initialization of the "where" clause of the SQL query, aiming to retrieve the output PostGIS layer - where = "" - # For each entity in the study area... - for feature in zone_etude.getFeatures(): - # Retrieve the geometry - area = feature.geometry() # QgsGeometry object - # Retrieve the geometry type (single or multiple) - geomSingleType = QgsWkbTypes.isSingleType(area.wkbType()) - # Increment the "where" clause - if geomSingleType: - where = where + "st_within(geom, ST_PolygonFromText('{}', 2154)) or ".format(area.asWkt()) - else: - where = where + "st_within(geom, ST_MPolyFromText('{}', 2154)) or ".format(area.asWkt()) - # Remove the last "or" in the "where" clause which is useless - where = where[:len(where)-4] - #feedback.pushInfo('Clause where : {}'.format(where)) + study_area = self.parameterAsSource(parameters, self.STUDY_AREA, context) + # Retrieve the output PostGIS layer name and format it + layer_name = self.parameterAsString(parameters, self.OUTPUT_NAME, context) + ts = datetime.now() + format_name = layer_name + " " + str(ts.strftime('%s')) + + # Construct the sql array containing the study area's features geometry + array_polygons = construct_sql_array_polygons(study_area) + # Define the "where" clause of the SQL query, aiming to retrieve the output PostGIS layer = biodiversity data + where = "is_valid and ST_within(geom, ST_union({}))".format(array_polygons) # Retrieve the data base connection name connection = self.parameterAsString(parameters, self.DATABASE, context) - # Retrieve the output PostGIS layer - # URI --> Configures connection to database and the SQL query + # URI --> Configures connection to database and the SQL query uri = postgis.uri_from_name(connection) uri.setDataSource("src_lpodatas", "observations", "geom", where) - layer_obs = QgsVectorLayer(uri.uri(), "Données d'observations {}".format(zone_etude.name()), "postgres") + # Retrieve the output PostGIS layer = biodiversity data + layer_obs = QgsVectorLayer(uri.uri(), format_name, "postgres") # Check if the PostGIS layer is valid - if not layer_obs.isValid(): - raise QgsProcessingException(self.tr("""Cette couche n'est pas valide ! - Checker les logs de PostGIS pour visualiser les messages d'erreur.""")) + check_layer_is_valid(feedback, layer_obs) + + # Create new valid fields for the sink + new_fields = format_layer_export(layer_obs) + # Retrieve the sink for the export + (sink, self.dest_id) = self.parameterAsSink(parameters, self.OUTPUT, context, new_fields, layer_obs.wkbType(), layer_obs.sourceCrs()) + if sink is None: + # Load the PostGIS layer and return it + load_layer(context, layer_obs) + return {self.OUTPUT: layer_obs.id()} else: - feedback.pushInfo('La couche PostGIS demandée est valide, la requête SQL a été exécutée avec succès !') - - # Load the PostGIS layer - root = context.project().layerTreeRoot() - plugin_lpo_group = root.findGroup('Résultats plugin LPO') - if not plugin_lpo_group: - plugin_lpo_group = root.insertGroup(0, 'Résultats plugin LPO') - context.project().addMapLayers([layer_obs], False) - plugin_lpo_group.addLayer(layer_obs) - # Variant - # context.temporaryLayerStore().addMapLayer(layer_obs) - # context.addLayerToLoadOnCompletion( - # layer_obs.id(), - # QgsProcessingContext.LayerDetails("Données d'observations", context.project(), self.OUTPUT) - # ) - - return {self.OUTPUT: layer_obs.id()} + # Fill the sink and return it + for feature in layer_obs.getFeatures(): + sink.addFeature(feature) + return {self.OUTPUT: self.dest_id} + + #def postProcessAlgorithm(self, context, feedback): + # processed_layer = QgsProcessingUtils.mapLayerFromString(self.dest_id, context) + # feedback.pushInfo('Processed_layer : {}'.format(processed_layer)) + #return {self.OUTPUT: self.dest_id} def tr(self, string): return QCoreApplication.translate('Processing', string) diff --git a/histogram.py b/histogram.py new file mode 100644 index 0000000..0868b61 --- /dev/null +++ b/histogram.py @@ -0,0 +1,235 @@ +# -*- coding: utf-8 -*- + +""" +/*************************************************************************** + ScriptsLPO : graph.py + ------------------- + Date : 2020-04-16 + Copyright : (C) 2020 by Elsa Guilley (LPO AuRA) + Email : lpo-aura@lpo.fr + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +__author__ = 'Elsa Guilley (LPO AuRA)' +__date__ = '2020-04-16' +__copyright__ = '(C) 2020 by Elsa Guilley (LPO AuRA)' + +# This will get replaced with a git SHA1 when you do a git archive +__revision__ = '$Format:%H$' + +import os +from datetime import datetime +from qgis.PyQt.QtGui import QIcon +from qgis.utils import iface + +# import plotly as plt +# import plotly.graph_objs as go +# import numpy as np + +from qgis.PyQt.QtCore import QCoreApplication +from qgis.core import (QgsProcessing, + QgsProcessingAlgorithm, + QgsProcessingParameterString, + QgsProcessingParameterFeatureSource, + QgsProcessingOutputVectorLayer, + QgsProcessingParameterBoolean, + QgsDataSourceUri, + QgsVectorLayer) +from processing.tools import postgis +from .common_functions import simplify_name, check_layer_is_valid, construct_sql_array_polygons, load_layer, execute_sql_queries + +pluginPath = os.path.dirname(__file__) + + +class Histogram(QgsProcessingAlgorithm): + """ + This algorithm takes a connection to a data base and a vector polygons layer and + returns a summary non geometric PostGIS layer. + """ + + # Constants used to refer to parameters and outputs + DATABASE = 'DATABASE' + STUDY_AREA = 'STUDY_AREA' + ADD_TABLE = 'ADD_TABLE' + OUTPUT = 'OUTPUT' + OUTPUT_NAME = 'OUTPUT_NAME' + + def name(self): + return 'Histogram' + + def displayName(self): + return 'Etat des connaissances par groupe taxonomique' + + def icon(self): + return QIcon(os.path.join(pluginPath, 'icons', 'table.png')) + + def groupId(self): + return 'test' + + def group(self): + return 'Test' + + def initAlgorithm(self, config=None): + """ + Here we define the inputs and output of the algorithm, along + with some other properties. + """ + + # Data base connection + db_param = QgsProcessingParameterString( + self.DATABASE, + self.tr("1/ Sélectionnez votre connexion à la base de données LPO AuRA") + ) + db_param.setMetadata( + { + 'widget_wrapper': {'class': 'processing.gui.wrappers_postgis.ConnectionWidgetWrapper'} + } + ) + self.addParameter(db_param) + + # Input vector layer = study area + self.addParameter( + QgsProcessingParameterFeatureSource( + self.STUDY_AREA, + self.tr("2/ Sélectionnez votre zone d'étude, à partir de laquelle seront extraites les données de l'état des connaissances"), + [QgsProcessing.TypeVectorPolygon] + ) + ) + + # Output PostGIS layer = histogram data + self.addOutput( + QgsProcessingOutputVectorLayer( + self.OUTPUT, + self.tr('Couche en sortie'), + QgsProcessing.TypeVectorAnyGeometry + ) + ) + + # Output PostGIS layer name + self.addParameter( + QgsProcessingParameterString( + self.OUTPUT_NAME, + self.tr("3/ Définissez un nom pour votre couche en sortie"), + self.tr("Etat des connaissances") + ) + ) + + # Boolean : True = add the summary table in the DB ; False = don't + self.addParameter( + QgsProcessingParameterBoolean( + self.ADD_TABLE, + self.tr("Enregistrer les données en sortie dans une nouvelle table PostgreSQL"), + False + ) + ) + + def processAlgorithm(self, parameters, context, feedback): + """ + Here is where the processing itself takes place. + """ + + # Retrieve the input vector layer = study area + study_area = self.parameterAsSource(parameters, self.STUDY_AREA, context) + # Retrieve the output PostGIS layer name and format it + layer_name = self.parameterAsString(parameters, self.OUTPUT_NAME, context) + ts = datetime.now() + format_name = layer_name + " " + str(ts.strftime('%s')) + + # Construct the sql array containing the study area's features geometry + array_polygons = construct_sql_array_polygons(study_area) + # Define the "where" clause of the SQL query, aiming to retrieve the output PostGIS layer = histogram data + where = "is_valid and ST_within(geom, ST_union({}))".format(array_polygons) + + # Retrieve the data base connection name + connection = self.parameterAsString(parameters, self.DATABASE, context) + # URI --> Configures connection to database and the SQL query + uri = postgis.uri_from_name(connection) + # Retrieve the boolean + add_table = self.parameterAsBool(parameters, self.ADD_TABLE, context) + + if add_table: + # Define the name of the PostGIS summary table which will be created in the DB + table_name = simplify_name(format_name) + # Define the SQL queries + queries = [ + "DROP TABLE if exists {}".format(table_name), + """CREATE TABLE {} AS (SELECT row_number() OVER () AS id, + groupe_taxo, COUNT(*) AS nb_donnees, + COUNT(DISTINCT(source_id_sp)) as nb_especes, + COUNT(DISTINCT(observateur)) as nb_observateurs, + COUNT(DISTINCT("date")) as nb_dates + FROM src_lpodatas.observations + WHERE {} + GROUP BY groupe_taxo + ORDER BY groupe_taxo)""".format(table_name, where), + "ALTER TABLE {} add primary key (id)".format(table_name) + ] + # Execute the SQL queries + execute_sql_queries(context, feedback, connection, queries) + # Format the URI + uri.setDataSource(None, table_name, None, "", "id") + + else: + # Define the SQL query + query = """(SELECT groupe_taxo, COUNT(*) AS nb_donnees, + COUNT(DISTINCT(source_id_sp)) as nb_especes, + COUNT(DISTINCT(observateur)) as nb_observateurs, + COUNT(DISTINCT("date")) as nb_dates + FROM src_lpodatas.observations + WHERE {} + GROUP BY groupe_taxo + ORDER BY groupe_taxo)""".format(where) + # Format the URI + uri.setDataSource("", query, None, "", "groupe_taxo") + + # Retrieve the output PostGIS layer = histogram data + layer_histo = QgsVectorLayer(uri.uri(), format_name, "postgres") + # Check if the PostGIS layer is valid + check_layer_is_valid(feedback, layer_histo) + # Load the PostGIS layer + load_layer(context, layer_histo) + # Open the attribute table of the PostGIS layer + iface.setActiveLayer(layer_histo) + iface.showAttributeTable(layer_histo) + + # x_var = [feature['groupe_taxo'] for feature in layer_histo.getFeatures()] + # y_var = [int(feature['nb_donnees']) for feature in layer_histo.getFeatures()] + # data = [go.Bar(x=x_var, y=y_var)] + # plt.offline.plot(data, filename="/home/eguilley/Téléchargements/histogram-test.html", auto_open=True) + # fig = go.Figure(data=data) + # fig.show() + # fig.write_image("/home/eguilley/Téléchargements/histogram-test.png") + + # plt.rcdefaults() + # libel = [feature['groupe_taxo'] for feature in layer_histo.getFeatures()] + # feedback.pushInfo('Libellés : {}'.format(libel)) + # #X = np.arange(len(libel)) + # #feedback.pushInfo('Valeurs en X : {}'.format(X)) + # Y = [int(feature['nb_observations']) for feature in layer_histo.getFeatures()] + # feedback.pushInfo('Valeurs en Y : {}'.format(Y)) + # fig = plt.figure() + # ax = fig.add_axes([0, 0, 1, 1]) + # # fig, ax = plt.subplots() + # ax.bar(libel, Y) + # # ax.set_xticks(X) + # ax.set_xticklabels(libel) + # ax.set_ylabel(u'Nombre d\'observations') + # ax.set_title(u'Etat des connaissances par groupes d\'espèces') + # plt.show() + + return {self.OUTPUT: layer_histo.id()} + + def tr(self, string): + return QCoreApplication.translate('Processing', string) + + def createInstance(self): + return Histogram() diff --git a/icons/extract_data.png b/icons/extract_data.png index 4f72703..3af85e2 100644 Binary files a/icons/extract_data.png and b/icons/extract_data.png differ diff --git a/icons/histogram.png b/icons/histogram.png new file mode 100644 index 0000000..5a4a6c8 Binary files /dev/null and b/icons/histogram.png differ diff --git a/icons/summary_table.png b/icons/table.png similarity index 100% rename from icons/summary_table.png rename to icons/table.png diff --git a/scripts_lpo_provider.py b/scripts_lpo_provider.py index 5834b46..fbe9819 100644 --- a/scripts_lpo_provider.py +++ b/scripts_lpo_provider.py @@ -32,6 +32,7 @@ from qgis.core import QgsProcessingProvider from .extract_data import ExtractData from .summary_table import SummaryTable +from .histogram import Histogram pluginPath = os.path.dirname(__file__) @@ -79,7 +80,7 @@ def loadAlgorithms(self): """ Loads all algorithms belonging to this provider. """ - algorithms = [ExtractData(), SummaryTable()] + algorithms = [ExtractData(), SummaryTable(), Histogram()] for algo in algorithms: self.addAlgorithm(algo) diff --git a/summary_table.py b/summary_table.py index a970b89..2b03a9b 100644 --- a/summary_table.py +++ b/summary_table.py @@ -27,23 +27,21 @@ __revision__ = '$Format:%H$' import os +from datetime import datetime from qgis.PyQt.QtGui import QIcon +from qgis.utils import iface from qgis.PyQt.QtCore import QCoreApplication from qgis.core import (QgsProcessing, QgsProcessingAlgorithm, QgsProcessingParameterString, - QgsProcessingParameterVectorLayer, + QgsProcessingParameterFeatureSource, QgsProcessingOutputVectorLayer, + QgsProcessingParameterBoolean, QgsDataSourceUri, - QgsVectorLayer, - QgsWkbTypes, - QgsProcessingContext, - QgsProcessingException) -from qgis.utils import iface + QgsVectorLayer) from processing.tools import postgis - -import processing +from .common_functions import simplify_name, check_layer_is_valid, construct_sql_array_polygons, load_layer, execute_sql_queries pluginPath = os.path.dirname(__file__) @@ -56,23 +54,25 @@ class SummaryTable(QgsProcessingAlgorithm): # Constants used to refer to parameters and outputs DATABASE = 'DATABASE' - ZONE_ETUDE = 'ZONE_ETUDE' + STUDY_AREA = 'STUDY_AREA' + ADD_TABLE = 'ADD_TABLE' OUTPUT = 'OUTPUT' + OUTPUT_NAME = 'OUTPUT_NAME' def name(self): return 'SummaryTable' def displayName(self): - return 'Create a summary table' + return 'Tableau de synthèse par espèce' def icon(self): - return QIcon(os.path.join(pluginPath, 'icons', 'summary_table.png')) + return QIcon(os.path.join(pluginPath, 'icons', 'table.png')) def groupId(self): - return 'treatments' + return 'test' def group(self): - return 'Treatments' + return 'Test' def initAlgorithm(self, config=None): """ @@ -83,7 +83,7 @@ def initAlgorithm(self, config=None): # Data base connection db_param = QgsProcessingParameterString( self.DATABASE, - self.tr('Nom de la connexion à la base de données') + self.tr("1/ Sélectionnez votre connexion à la base de données LPO AuRA") ) db_param.setMetadata( { @@ -94,14 +94,14 @@ def initAlgorithm(self, config=None): # Input vector layer = study area self.addParameter( - QgsProcessingParameterVectorLayer( - self.ZONE_ETUDE, - self.tr("Zone d'étude"), - [QgsProcessing.TypeVectorAnyGeometry] + QgsProcessingParameterFeatureSource( + self.STUDY_AREA, + self.tr("2/ Sélectionnez votre zone d'étude, à partir de laquelle seront extraites les données du tableau de synthèse"), + [QgsProcessing.TypeVectorPolygon] ) ) - # Output PostGIS layer + # Output PostGIS layer = summary table self.addOutput( QgsProcessingOutputVectorLayer( self.OUTPUT, @@ -110,95 +110,120 @@ def initAlgorithm(self, config=None): ) ) + # Output PostGIS layer name + self.addParameter( + QgsProcessingParameterString( + self.OUTPUT_NAME, + self.tr("3/ Définissez un nom pour votre couche en sortie"), + self.tr("Tableau synthèse") + ) + ) + + # Boolean : True = add the summary table in the DB ; False = don't + self.addParameter( + QgsProcessingParameterBoolean( + self.ADD_TABLE, + self.tr("Enregistrer les données en sortie dans une nouvelle table PostgreSQL"), + False + ) + ) + def processAlgorithm(self, parameters, context, feedback): """ Here is where the processing itself takes place. """ # Retrieve the input vector layer = study area - zone_etude = self.parameterAsVectorLayer(parameters, self.ZONE_ETUDE, context) - # Define the name of the PostGIS summary table which will be created in the DB - table_name = "summary_table_{}".format(zone_etude.name()) - # Define the name of the output PostGIS layer (summary table) which will be loaded in the QGis project - layer_name = "Tableau synthèse {}".format(zone_etude.name()) - - # Initialization of the "where" clause of the SQL query, aiming to create the summary table in the DB - where = "and (" - # For each entity in the study area... - for feature in zone_etude.getFeatures(): - # Retrieve the geometry - area = feature.geometry() # QgsGeometry object - # Retrieve the geometry type (single or multiple) - geomSingleType = QgsWkbTypes.isSingleType(area.wkbType()) - # Increment the "where" clause - if geomSingleType: - where = where + "st_within(geom, ST_PolygonFromText('{}', 2154)) or ".format(area.asWkt()) - else: - where = where + "st_within(geom, ST_MPolyFromText('{}', 2154)) or ".format(area.asWkt()) - # Remove the last "or" in the "where" clause which is useless - where = where[:len(where)-4] + ")" - #feedback.pushInfo('Clause where : {}'.format(where)) - - # Define the SQL queries - queries = [ - "drop table if exists {}".format(table_name), - """create table {} as ( - select row_number() OVER () AS id, source_id_sp, nom_sci, nom_vern, - count(*) as nb_observations, count(distinct(observateur)) as nb_observateurs, max(date_an) as derniere_observation - from src_lpodatas.observations - where is_valid {} - group by source_id_sp, nom_sci, nom_vern - order by source_id_sp)""".format(table_name, where), - "alter table {} add primary key (id)".format(table_name) - ] - + study_area = self.parameterAsSource(parameters, self.STUDY_AREA, context) + # Retrieve the output PostGIS layer name and format it + layer_name = self.parameterAsString(parameters, self.OUTPUT_NAME, context) + ts = datetime.now() + format_name = layer_name + " " + str(ts.strftime('%s')) + + # Construct the sql array containing the study area's features geometry + array_polygons = construct_sql_array_polygons(study_area) + # Define the "where" clause of the SQL query, aiming to retrieve the output PostGIS layer = summary table + where = "is_valid and ST_within(geom, ST_union({}))".format(array_polygons) + # Retrieve the data base connection name connection = self.parameterAsString(parameters, self.DATABASE, context) - # Execute the SQL queries - for query in queries: - processing.run( - 'qgis:postgisexecutesql', - { - 'DATABASE': connection, - 'SQL': query - }, - is_child_algorithm=True, - context=context, - feedback=feedback - ) - feedback.pushInfo('Requête SQL exécutée avec succès !') - - # Retrieve the output PostGIS layer (summary table) which has just been created - # URI --> Configures connection to database and the SQL query + # URI --> Configures connection to database and the SQL query uri = postgis.uri_from_name(connection) - uri.setDataSource(None, table_name, None, "", "id") - layer_summary = QgsVectorLayer(uri.uri(), layer_name, "postgres") + # Retrieve the boolean + add_table = self.parameterAsBool(parameters, self.ADD_TABLE, context) + + if add_table: + # Define the name of the PostGIS summary table which will be created in the DB + table_name = simplify_name(format_name) + # Define the SQL queries + queries = [ + "DROP TABLE if exists {}".format(table_name), + """CREATE TABLE {} AS (WITH data AS + (SELECT source_id_sp, nom_sci AS nom_scientifique, nom_vern AS nom_vernaculaire, groupe_taxo, + COUNT(*) AS nb_donnees, COUNT(DISTINCT(observateur)) AS nb_observateurs, + COUNT(DISTINCT("date")) as nb_dates, + COALESCE(SUM(CASE WHEN mortalite THEN 1 ELSE 0 END),0) AS nb_mortalite, + max(sn.code_nidif) AS max_atlas_code, max(nombre_total) AS nb_individus_max, + min (date_an) as premiere_observation, max(date_an) as derniere_observation, + string_agg(distinct source,', ') as sources + FROM src_lpodatas.observations obs + LEFT JOIN referentiel.statut_nidif sn ON obs.oiso_code_nidif = sn.code_repro + WHERE {} + GROUP BY source_id_sp, nom_sci, nom_vern, groupe_taxo), + synthese AS + (SELECT DISTINCT source_id_sp, nom_scientifique, nom_vernaculaire, groupe_taxo, + nb_donnees, nb_observateurs, nb_dates, nb_mortalite, + sn2.statut_nidif, nb_individus_max, + premiere_observation, derniere_observation, sources + FROM data d + LEFT JOIN referentiel.statut_nidif sn2 ON d.max_atlas_code = sn2.code_nidif + ORDER BY groupe_taxo, source_id_sp) + SELECT row_number() OVER () AS id, * + FROM synthese)""".format(table_name, where), + "ALTER TABLE {} add primary key (id)".format(table_name) + ] + # Execute the SQL queries + execute_sql_queries(context, feedback, connection, queries) + # Format the URI + uri.setDataSource(None, table_name, None, "", "id") - # Check if the PostGIS layer is valid - if not layer_summary.isValid(): - raise QgsProcessingException(self.tr("""Cette couche n'est pas valide ! - Checker les logs de PostGIS pour visualiser les messages d'erreur.""")) else: - feedback.pushInfo('La couche PostGIS demandée est valide, la requête SQL a été exécutée avec succès !') - + # Define the SQL query + query = """(WITH data AS + (SELECT source_id_sp, nom_sci AS nom_scientifique, nom_vern AS nom_vernaculaire, groupe_taxo, + COUNT(*) AS nb_donnees, COUNT(DISTINCT(observateur)) AS nb_observateurs, + COUNT(DISTINCT("date")) as nb_dates, + COALESCE(SUM(CASE WHEN mortalite THEN 1 ELSE 0 END),0) AS nb_mortalite, + max(sn.code_nidif) AS max_atlas_code, max(nombre_total) AS nb_individus_max, + min (date_an) as premiere_observation, max(date_an) as derniere_observation, + string_agg(distinct source,', ') as sources + FROM src_lpodatas.observations obs + LEFT JOIN referentiel.statut_nidif sn ON obs.oiso_code_nidif = sn.code_repro + WHERE {} + GROUP BY source_id_sp, nom_sci, nom_vern, groupe_taxo), + synthese AS + (SELECT DISTINCT source_id_sp, nom_scientifique, nom_vernaculaire, groupe_taxo, + nb_donnees, nb_observateurs, nb_dates, nb_mortalite, + sn2.statut_nidif, nb_individus_max, + premiere_observation, derniere_observation, sources + FROM data d + LEFT JOIN referentiel.statut_nidif sn2 ON d.max_atlas_code = sn2.code_nidif + ORDER BY groupe_taxo, source_id_sp) + SELECT row_number() OVER () AS id, * + FROM synthese)""".format(where) + # Format the URI + uri.setDataSource("", query, None, "", "id") + + # Retrieve the output PostGIS layer = summary table + layer_summary = QgsVectorLayer(uri.uri(), format_name, "postgres") + # Check if the PostGIS layer is valid + check_layer_is_valid(feedback, layer_summary) # Load the PostGIS layer - root = context.project().layerTreeRoot() - plugin_lpo_group = root.findGroup('Résultats plugin LPO') - if not plugin_lpo_group: - plugin_lpo_group = root.insertGroup(0, 'Résultats plugin LPO') - context.project().addMapLayers([layer_summary], False) - plugin_lpo_group.addLayer(layer_summary) - # Variant - # context.temporaryLayerStore().addMapLayer(layer_summary) - # context.addLayerToLoadOnCompletion( - # layer_summary.id(), - # QgsProcessingContext.LayerDetails(layer_name, context.project(), self.OUTPUT) - # ) - + load_layer(context, layer_summary) # Open the attribute table of the PostGIS layer iface.setActiveLayer(layer_summary) iface.showAttributeTable(layer_summary) - + return {self.OUTPUT: layer_summary.id()} def tr(self, string):