Merge branch 'develop' into patch-4

.github/workflows/cypress.yml

@@ -5,12 +5,13 @@ on:
       - master
       - hotfixes
       - develop
-      - fixtestfront
+      - feat/import
   pull_request:
     branches:
       - master
       - hotfixes
       - develop
+      - feat/import
 
 jobs:
   mount_app_and_run_cypress:
@@ -31,6 +32,15 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
     steps:
       - name: Add postgis_raster database extension
@@ -78,6 +88,7 @@ jobs:
           GEONATURE_CONFIG_FILE: config/test_config.toml
           srid_local: 2154
           install_bdc_statuts: true
+          taxref_region: fr
           add_sample_data: true
           install_sig_layers: true
           install_grid_layer_5: true
@@ -100,7 +111,7 @@ jobs:
           cp ./config/settings.ini.sample ./config/settings.ini 
           ./install/05_install_frontend.sh --ci
         env:
-          GEONATURE_CONFIG_FILE: "${{ github.workspace }}/config/test_config.toml"
+          GEONATURE_CONFIG_FILE: '${{ github.workspace }}/config/test_config.toml'
       - name: Install core modules
         run: |
           geonature install-gn-module contrib/occtax OCCTAX --build=false
@@ -109,18 +120,18 @@ jobs:
           geonature db upgrade occhab-samples@head
           geonature install-gn-module contrib/gn_module_validation VALIDATION --build=false
           geonature permissions supergrant --group --nom "Grp_admin" --yes
+          geonature db upgrade import-samples@head
         env:
           GEONATURE_CONFIG_FILE: config/test_config.toml
       - name: Run GeoNature backend
         run: geonature dev_back &
         env:
           GEONATURE_CONFIG_FILE: config/test_config.toml
-      - name: Run TaxHub backend
-        run: flask run --host=0.0.0.0 &
-        working-directory: ./backend/dependencies/TaxHub/
+      - name: Run celery
+        run: celery -A geonature.celery_app:app worker &
+        working-directory: ./backend/geonature/
         env:
-          TAXHUB_SETTINGS: test_config.py
-          TAXHUB_SQLALCHEMY_DATABASE_URI: "postgresql://geonatadmin:geonatpasswd@127.0.0.1:5432/geonature2db"
+          GEONATURE_CONFIG_FILE: "${{ github.workspace }}/config/test_config.toml"
       - name: Cypress run
         uses: cypress-io/github-action@v5
         with:

.github/workflows/eval.yml

@@ -94,6 +94,7 @@ jobs:
           GEONATURE_CONFIG_FILE: config/test_config.toml
           srid_local: 2154
           install_bdc_statuts: true
+          taxref_region: fr
           add_sample_data: true
           install_sig_layers: true
           install_grid_layer_5: true

.github/workflows/eval_perf.yml

@@ -9,16 +9,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        debian-version: ["11", "12"]
+        debian-version: ['11', '12']
         include:
-          - debian-version: "11"
-            python-version: "3.9"
-            postgres-version: "13"
-            postgis-version: "3.2"
-          - debian-version: "12"
-            python-version: "3.11"
-            postgres-version: "15"
-            postgis-version: "3.3"
+          - debian-version: '11'
+            python-version: '3.9'
+            postgres-version: '13'
+            postgis-version: '3.2'
+          - debian-version: '12'
+            python-version: '3.11'
+            postgres-version: '15'
+            postgis-version: '3.3'
 
     name: Debian ${{ matrix.debian-version }}
 
@@ -50,7 +50,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
+          cache: 'pip'
       - name: Install GDAL
         run: |
           sudo apt update
@@ -94,6 +94,7 @@ jobs:
           GEONATURE_CONFIG_FILE: config/test_config.toml
           srid_local: 2154
           install_bdc_statuts: true
+          taxref_region: fr
           add_sample_data: true
           install_sig_layers: true
           install_grid_layer_5: true

.github/workflows/pytest.yml

@@ -15,7 +15,6 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-
     strategy:
       fail-fast: false
       matrix:
@@ -46,6 +45,15 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
     steps:
       - uses: actions/checkout@v4
@@ -95,6 +103,7 @@ jobs:
           GEONATURE_CONFIG_FILE: config/test_config.toml
           srid_local: 2154
           install_bdc_statuts: true
+          taxref_region: fr
           add_sample_data: true
           install_sig_layers: true
           install_grid_layer_5: true
@@ -122,11 +131,12 @@ jobs:
           GEONATURE_CONFIG_FILE: config/test_config.toml
       - name: Test with pytest
         run: |
-          pytest -v --cov --cov-report xml --benchmark-skip
+          pytest -v --cov --cov-report xml
         env:
           GEONATURE_CONFIG_FILE: config/test_config.toml
       - name: Upload coverage to Codecov
-        if: ${{ matrix.debian-version == '11' }}
-        uses: codecov/codecov-action@v3
+        if: ${{ matrix.debian-version == '12' }}
+        uses: codecov/codecov-action@v4
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           flags: pytest

.gitignore

@@ -30,6 +30,11 @@ cache/*
 .angulardoc.json
 Pipfile.lock
 
+custom/*
+!custom/.gitkeep
+
+environ
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

backend/geonature/app.py

@@ -11,40 +11,35 @@
     from importlib_metadata import entry_points
 else:
     from importlib.metadata import entry_points
-
 from flask import Flask, g, request, current_app, send_from_directory
 from flask.json.provider import DefaultJSONProvider
 from flask_mail import Message
+from flask_babel import Babel
 from flask_cors import CORS
 from flask_login import current_user
 from flask_sqlalchemy.track_modifications import before_models_committed
 from werkzeug.middleware.proxy_fix import ProxyFix
 from werkzeug.middleware.shared_data import SharedDataMiddleware
 from werkzeug.middleware.dispatcher import DispatcherMiddleware
 from werkzeug.wrappers import Response
-from psycopg2.errors import UndefinedTable
 import sqlalchemy as sa
-from sqlalchemy.exc import OperationalError, ProgrammingError
-from sqlalchemy.orm.exc import NoResultFound
 
 if version.parse(sa.__version__) >= version.parse("1.4"):
     from sqlalchemy.engine import Row
 else:  # retro-compatibility SQLAlchemy 1.3
     from sqlalchemy.engine import RowProxy as Row
 
 from geonature.utils.config import config
+
 from geonature.utils.env import MAIL, DB, db, MA, migrate, BACKEND_DIR
 from geonature.utils.logs import config_loggers
 from geonature.utils.module import iter_modules_dist
 from geonature.core.admin.admin import admin
 from geonature.middlewares import SchemeFix, RequestID
 
-from pypnusershub.db.tools import (
-    user_from_token,
-    UnreadableAccessRightsError,
-    AccessRightsExpiredError,
-)
+
 from pypnusershub.db.models import Application
+from pypnusershub.auth import auth_manager
 from pypnusershub.login_manager import login_manager
 
 
@@ -88,6 +83,17 @@ def default(o):
         return DefaultJSONProvider.default(o)
 
 
+def get_locale():
+    # if a user is logged in, use the locale from the user settings
+    user = getattr(g, "user", None)
+    if user is not None:
+        return user.locale
+    # otherwise try to guess the language from the user accept
+    # header the browser transmits.  We support de/fr/en in this
+    # example.  The best match wins.
+    return request.accept_languages.best_match(["de", "fr", "en"])
+
+
 def create_app(with_external_mods=True):
     app = Flask(
         __name__.split(".")[0],
@@ -96,7 +102,6 @@ def create_app(with_external_mods=True):
         static_url_path=config["STATIC_URL"],
         template_folder="geonature/templates",
     )
-
     app.config.update(config)
 
     # Enable deprecation warnings in debug mode
@@ -129,12 +134,13 @@ def create_app(with_external_mods=True):
     migrate.init_app(app, DB, directory=BACKEND_DIR / "geonature" / "migrations")
     MA.init_app(app)
     CORS(app, supports_credentials=True)
+    auth_manager.init_app(app, providers_declaration=config["AUTHENTICATION"]["PROVIDERS"])
+    auth_manager.home_page = config["URL_APPLICATION"]
 
     if "CELERY" in app.config:
         from geonature.utils.celery import celery_app
 
         celery_app.init_app(app)
-        celery_app.conf.update(app.config["CELERY"])
 
     # Emails configuration
     if app.config["MAIL_CONFIG"]:
@@ -148,8 +154,6 @@ def create_app(with_external_mods=True):
     # Pass parameters to the submodules
     app.config["MA"] = MA
 
-    login_manager.init_app(app)
-
     # For deleting files on "delete" media
     @before_models_committed.connect_via(app)
     def on_before_models_committed(sender, changes):
@@ -182,15 +186,24 @@ def set_sentry_context():
 
     admin.init_app(app)
 
+    # babel
+    babel = Babel(app, locale_selector=get_locale)
+
     # Enable serving of media files
     app.add_url_rule(
         f"{config['MEDIA_URL']}/<path:filename>",
         view_func=lambda filename: send_from_directory(config["MEDIA_FOLDER"], filename),
         endpoint="media",
     )
+    app.add_url_rule(
+        f"{config['MEDIA_URL']}/taxhub/<path:filename>",
+        view_func=lambda filename: send_from_directory(
+            config["MEDIA_FOLDER"] + "/taxhub", filename
+        ),
+        endpoint="media_taxhub",
+    )
 
     for blueprint_path, url_prefix in [
-        ("pypnusershub.routes:routes", "/auth"),
         ("pypn_habref_api.routes:routes", "/habref"),
         ("pypnusershub.routes_register:bp", "/pypn/register"),
         ("pypnnomenclature.routes:routes", "/nomenclatures"),
@@ -200,17 +213,37 @@ def set_sentry_context():
         ("geonature.core.users.routes:routes", "/users"),
         ("geonature.core.gn_synthese.routes:routes", "/synthese"),
         ("geonature.core.gn_meta.routes:routes", "/meta"),
-        ("geonature.core.auth.routes:routes", "/gn_auth"),
         ("geonature.core.gn_monitoring.routes:routes", "/gn_monitoring"),
         ("geonature.core.gn_profiles.routes:routes", "/gn_profiles"),
         ("geonature.core.sensitivity.routes:routes", None),
         ("geonature.core.notifications.routes:routes", "/notifications"),
+        ("geonature.core.imports.blueprint:blueprint", "/import"),
     ]:
         module_name, blueprint_name = blueprint_path.split(":")
         blueprint = getattr(import_module(module_name), blueprint_name)
         app.register_blueprint(blueprint, url_prefix=url_prefix)
 
     with app.app_context():
+        # taxhub api
+        from apptax import taxhub_api_routes
+
+        base_api_prefix = app.config["TAXHUB"].get("API_PREFIX")
+
+        for blueprint_path, url_prefix in taxhub_api_routes:
+            module_name, blueprint_name = blueprint_path.split(":")
+            blueprint = getattr(import_module(module_name), blueprint_name)
+            app.register_blueprint(blueprint, url_prefix="/taxhub" + base_api_prefix + url_prefix)
+
+        # taxhub admin
+        from apptax.admin.admin import adresses
+
+        app.register_blueprint(adresses, url_prefix="/taxhub")
+
+        # register taxhub admin view which need app context
+        from geonature.core.taxonomie.admin import load_admin_views
+
+        load_admin_views(app, admin)
+
         # register errors handlers
         import geonature.core.errors
 

backend/geonature/celery_app.py

@@ -1,6 +1,7 @@
 from .app import create_app
 from .utils.celery import celery_app as app
 from .utils.module import iter_modules_dist
+from .utils.env import db
 
 
 flask_app = create_app()
@@ -9,7 +10,9 @@
 class ContextTask(app.Task):
     def __call__(self, *args, **kwargs):
         with flask_app.app_context():
-            return self.run(*args, **kwargs)
+            result = self.run(*args, **kwargs)
+            db.session.remove()
+        return result
 
 
 app.Task = ContextTask

backend/geonature/core/admin/admin.py

@@ -1,3 +1,5 @@
+import os
+
 from flask import g
 from werkzeug.exceptions import Unauthorized
 from flask_admin import Admin, AdminIndexView, expose
@@ -152,4 +154,5 @@ class ProtectedTNomenclaturesAdmin(
     )
 )
 
+
 flask_admin = admin  # for retro-compatibility, usefull for export module for instance

backend/geonature/core/admin/utils.py

@@ -10,6 +10,8 @@ class CruvedProtectedMixin:
     def is_accessible(self):
         if g.current_user is None:
             raise Unauthorized  # return False leads to Forbidden which is different
+        if not g.current_user.is_authenticated:
+            raise Unauthorized
         return self._can_action("R")
 
     def _can_action(self, action):