Use marshalow schema to validate synthese get_observations_for_web view

remove use of json_schema
fix PnX-SI#2907

backend/geonature/core/gn_synthese/models.py

@@ -586,7 +586,7 @@ class VSyntheseForWebApp(DB.Model):
     id_module = DB.Column(DB.Integer)
     entity_source_pk_value = DB.Column(DB.Integer)
     id_dataset = DB.Column(DB.Integer)
-    dataset_name = DB.Column(DB.Integer)
+    dataset_name = DB.Column(DB.String)
     id_acquisition_framework = DB.Column(DB.Integer)
     count_min = DB.Column(DB.Integer)
     count_max = DB.Column(DB.Integer)

backend/geonature/core/gn_synthese/routes.py

@@ -16,6 +16,7 @@
     g,
 )
 from geonature.core.gn_synthese.schemas import SyntheseSchema
+from geonature.core.gn_synthese.synthese_config import MANDATORY_COLUMNS
 from pypnusershub.db.models import User
 from pypnnomenclature.models import BibNomenclaturesTypes, TNomenclatures
 from werkzeug.exceptions import Forbidden, NotFound, BadRequest, Conflict
@@ -151,12 +152,9 @@ def get_observations_for_web(permissions):
         col["prop"] for col in current_app.config["SYNTHESE"]["LIST_COLUMNS_FRONTEND"]
     }
     # Init with compulsory columns
-    columns = [
-        "id",
-        VSyntheseForWebApp.id_synthese,
-        "url_source",
-        VSyntheseForWebApp.url_source,
-    ]
+    columns = []
+    for col in MANDATORY_COLUMNS:
+        columns.extend([col, getattr(VSyntheseForWebApp, col)])
 
     if "count_min_max" in param_column_list:
         count_min_max = case(

backend/geonature/core/gn_synthese/schemas.py

@@ -1,7 +1,15 @@
 from geonature.utils.env import db, ma
+from geonature.utils.config import config
 
 from geonature.core.gn_commons.schemas import ModuleSchema, MediaSchema, TValidationSchema
-from geonature.core.gn_synthese.models import BibReportsTypes, TReport, TSources, Synthese
+from geonature.core.gn_synthese.models import (
+    BibReportsTypes,
+    TReport,
+    TSources,
+    Synthese,
+    VSyntheseForWebApp,
+)
+from geonature.core.gn_synthese.synthese_config import MANDATORY_COLUMNS
 
 from pypn_habref_api.schemas import HabrefSchema
 from pypnusershub.schemas import UserSchema
@@ -62,3 +70,24 @@ class Meta:
     last_validation = ma.Nested(TValidationSchema, dump_only=True)
     reports = ma.Nested(ReportSchema, many=True)
     # Missing nested schemas: taxref
+
+
+class CustomRequiredConverter(GeoModelConverter):
+    """Custom converter to add kwargs required for mandatory and asked fields in get_observations_for_web view
+    Use to validate response in test"""
+
+    def _add_column_kwargs(self, kwargs, column):
+        super()._add_column_kwargs(kwargs, column)
+        default_cols = map(lambda col: col["prop"], config["SYNTHESE"]["LIST_COLUMNS_FRONTEND"])
+        required_cols = list(default_cols) + MANDATORY_COLUMNS
+        kwargs["required"] = column.name in required_cols
+
+
+# Only used in test for now
+class VSyntheseForWebAppSchema(GeoAlchemyAutoSchema):
+
+    class Meta:
+        model = VSyntheseForWebApp
+        feature_geometry = "the_geom_4326"
+        sqla_session = db.session
+        model_converter = CustomRequiredConverter

backend/geonature/core/gn_synthese/synthese_config.py

@@ -91,7 +91,7 @@
 ]
 
 # Mandatory columns for the frontend in Synthese API
-MANDATORY_COLUMNS = ["entity_source_pk_value", "url_source", "cd_nom"]
+MANDATORY_COLUMNS = ["id_synthese", "entity_source_pk_value", "url_source", "cd_nom"]
 
 # CONFIG MAP-LIST
 DEFAULT_LIST_COLUMN = [

backend/geonature/tests/test_synthese.py

@@ -14,7 +14,9 @@
 from geoalchemy2.shape import to_shape, from_shape
 from shapely.testing import assert_geometries_equal
 from shapely.geometry import Point
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, fields, Schema
+from marshmallow_geojson import FeatureSchema, GeoJSONSchema
+
 
 from geonature.utils.env import db
 from geonature.core.gn_permissions.tools import get_permissions
@@ -23,7 +25,7 @@
 from geonature.core.sensitivity.models import cor_sensitivity_area_type
 from geonature.core.gn_meta.models import TDatasets
 from geonature.core.gn_synthese.models import Synthese, TSources, VSyntheseForWebApp
-from geonature.core.gn_synthese.schemas import SyntheseSchema
+from geonature.core.gn_synthese.schemas import SyntheseSchema, VSyntheseForWebAppSchema
 from geonature.core.gn_permissions.models import PermAction, Permission
 from geonature.core.gn_commons.models.base import TModules
 
@@ -34,7 +36,6 @@
 
 from .fixtures import *
 from .fixtures import create_synthese, create_module, synthese_with_protected_status
-from .utils import jsonschema_definitions
 
 
 @pytest.fixture()
@@ -106,59 +107,48 @@ def synthese_for_observers(source, datasets):
             )
 
 
-synthese_properties = {
-    "type": "object",
-    "properties": {
-        "observations": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "id": {"type": "number"},
-                    "cd_nom": {"type": "number"},
-                    "count_min_max": {"type": "string"},
-                    "dataset_name": {"type": "string"},
-                    "date_min": {"type": "string"},
-                    "entity_source_pk_value": {
-                        "oneOf": [
-                            {"type": "null"},
-                            {"type": "string"},
-                        ],
-                    },
-                    "lb_nom": {"type": "string"},
-                    "nom_vern_or_lb_nom": {"type": "string"},
-                    "unique_id_sinp": {
-                        "type": "string",
-                        "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$",
-                    },
-                    "observers": {
-                        "oneOf": [
-                            {"type": "null"},
-                            {"type": "string"},
-                        ],
-                    },
-                    "url_source": {
-                        "oneOf": [
-                            {"type": "null"},
-                            {"type": "string"},
-                        ],
-                    },
-                },
-                "required": [  # obligatoire pour le fonctionement du front
-                    "id",
-                    "cd_nom",
-                    "url_source",
-                    "entity_source_pk_value",
-                ],
-                # "additionalProperties": False,
-            },
-        },
-    },
-}
+# utility classes for VSyntheseForWebAppSchema validation
+class UngroupedFeatureSchema(FeatureSchema):
+    properties = fields.Nested(
+        VSyntheseForWebAppSchema,
+        required=True,
+    )
+
+
+class NestedObs(Schema):
+    observations = fields.List(
+        fields.Nested(VSyntheseForWebAppSchema, required=True), required=True
+    )
+
+
+class GroupedFeatureSchema(FeatureSchema):
+    properties = fields.Nested(NestedObs, required=True)
+
+
+class UngroupedGeoJSONSchema(GeoJSONSchema):
+    feature_schema = UngroupedFeatureSchema
+
+
+class GroupedGeoJSONSchema(GeoJSONSchema):
+    feature_schema = GroupedFeatureSchema
 
 
 @pytest.mark.usefixtures("client_class", "temporary_transaction")
 class TestSynthese:
+    def test_required_fields_and_format(self, app, users):
+        # Test required fields base on VSyntheseForWebAppSchema surrounded by a custom converter : CustomRequiredConverter
+        # also test geojson serialization (grouped by geometry and not)
+        url_ungrouped = url_for("gn_synthese.get_observations_for_web")
+        set_logged_user(self.client, users["admin_user"])
+        resp = self.client.get(url_ungrouped)
+        for f in resp.json["features"]:
+            UngroupedGeoJSONSchema().load(f)
+
+        url_grouped = url_for("gn_synthese.get_observations_for_web", format="grouped_geom")
+        resp = self.client.get(url_grouped)
+        for f in resp.json["features"]:
+            GroupedGeoJSONSchema().load(f)
+
     def test_synthese_scope_filtering(self, app, users, synthese_data):
         all_ids = {s.id_synthese for s in synthese_data.values()}
         sq = (
@@ -184,20 +174,16 @@ def test_get_defaut_nomenclatures(self, users):
 
     def test_get_observations_for_web(self, app, users, synthese_data, taxon_attribut):
         url = url_for("gn_synthese.get_observations_for_web")
-        schema = {
-            "definitions": jsonschema_definitions,
-            "$ref": "#/definitions/featurecollection",
-            "$defs": {"props": synthese_properties},
-        }
-
         r = self.client.get(url)
         assert r.status_code == Unauthorized.code
 
         set_logged_user(self.client, users["self_user"])
 
         r = self.client.get(url)
         assert r.status_code == 200
-        validate_json(instance=r.json, schema=schema)
+
+        r = self.client.get(url)
+        assert r.status_code == 200
 
         # Add cd_nom column
         app.config["SYNTHESE"]["LIST_COLUMNS_FRONTEND"] += [
@@ -215,7 +201,6 @@ def test_get_observations_for_web(self, app, users, synthese_data, taxon_attribu
         }
         r = self.client.post(url, json=filters)
         assert r.status_code == 200
-        validate_json(instance=r.json, schema=schema)
         assert len(r.json["features"]) > 0
         for feature in r.json["features"]:
             assert feature["properties"]["cd_nom"] == taxon_attribut.bib_nom.cd_nom
@@ -241,7 +226,6 @@ def test_get_observations_for_web(self, app, users, synthese_data, taxon_attribu
         }
         r = self.client.post(url, json=filters)
         assert r.status_code == 200
-        validate_json(instance=r.json, schema=schema)
         assert {synthese_data[k].id_synthese for k in ["p1_af1", "p1_af2"]}.issubset(
             {f["properties"]["id"] for f in r.json["features"]}
         )
@@ -264,7 +248,6 @@ def test_get_observations_for_web(self, app, users, synthese_data, taxon_attribu
         }
         r = self.client.post(url, json=filters)
         assert r.status_code == 200
-        validate_json(instance=r.json, schema=schema)
         assert {synthese_data[k].id_synthese for k in ["p1_af1", "p1_af2"]}.issubset(
             {f["properties"]["id"] for f in r.json["features"]}
         )
@@ -280,7 +263,6 @@ def test_get_observations_for_web(self, app, users, synthese_data, taxon_attribu
         filters = {f"area_{com_type.id_type}": [chambery.id_area]}
         r = self.client.post(url, json=filters)
         assert r.status_code == 200
-        validate_json(instance=r.json, schema=schema)
         assert {synthese_data[k].id_synthese for k in ["p1_af1", "p1_af2"]}.issubset(
             {f["properties"]["id"] for f in r.json["features"]}
         )
@@ -294,7 +276,6 @@ def test_get_observations_for_web(self, app, users, synthese_data, taxon_attribu
         }
         r = self.client.post(url, json=filters)
         assert r.status_code == 200
-        validate_json(instance=r.json, schema=schema)
         assert len(r.json["features"]) >= 2  # FIXME
 
         # test status lr

backend/geonature/tests/utils.py

@@ -29,79 +29,3 @@ def get_id_nomenclature(nomenclature_type_mnemonique, cd_nomenclature):
             )
         )
     )
-
-
-jsonschema_definitions = {
-    "geometries": {
-        "BoundingBox": {
-            "type": "array",
-            "minItems": 4,
-            "items": {"type": "number"},
-        },
-        "PointCoordinates": {"type": "array", "minItems": 2, "items": {"type": "number"}},
-        "Point": {
-            "title": "GeoJSON Point",
-            "type": "object",
-            "required": ["type", "coordinates"],
-            "properties": {
-                "type": {"type": "string", "enum": ["Point"]},
-                "coordinates": {
-                    "$ref": "#/definitions/geometries/PointCoordinates",
-                },
-                "bbox": {
-                    "$ref": "#/definitions/geometries/BoundingBox",
-                },
-            },
-        },
-    },
-    "feature": {
-        "title": "GeoJSON Feature",
-        "type": "object",
-        "required": ["type", "properties", "geometry"],
-        "properties": {
-            "type": {"type": "string", "enum": ["Feature"]},
-            "id": {"oneOf": [{"type": "number"}, {"type": "string"}]},
-            "properties": {
-                "oneOf": [
-                    {"type": "null"},
-                    {"$ref": "#/$defs/props"},
-                ],
-            },
-            "geometry": {
-                "oneOf": [
-                    {"type": "null"},
-                    {"$ref": "#/definitions/geometries/Point"},
-                    # {"$ref": "#/definitions/geometries/LineString"},
-                    # {"$ref": "#/definitions/geometries/Polygon"},
-                    # {"$ref": "#/definitions/geometries/MultiPoint"},
-                    # {"$ref": "#/definitions/geometries/MultiLineString"},
-                    # {"$ref": "#/definitions/geometries/MultiPolygon"},
-                    # {"$ref": "#/definitions/geometries/GeometryCollection"},
-                ],
-            },
-            "bbox": {
-                "$ref": "#/definitions/geometries/BoundingBox",
-            },
-        },
-    },
-    "featurecollection": {
-        "title": "GeoJSON FeatureCollection",
-        "type": "object",
-        "required": ["type", "features"],
-        "properties": {
-            "type": {
-                "type": "string",
-                "enum": ["FeatureCollection"],
-            },
-            "features": {
-                "type": "array",
-                "items": {
-                    "$ref": "#/definitions/feature",
-                },
-            },
-            "bbox": {
-                "$ref": "#/definitions/geometries/BoundingBox",
-            },
-        },
-    },
-}