Merge branch 'master' into 10995_faceting_fixes

geonode/base/admin.py

@@ -338,7 +338,7 @@ def keyword_id(self, obj):
 
 
 class ResourceBaseAdminForm(autocomplete.FutureModelForm):
-    keywords = TagField(widget=TaggitSelect2Custom("autocomplete_hierachical_keyword"))
+    keywords = TagField(widget=TaggitSelect2Custom("autocomplete_hierachical_keyword"), required=False)
 
     def delete_queryset(self, request, queryset):
         """

geonode/base/auth.py

@@ -259,6 +259,8 @@ def visitor_ip_address(request):
         ip = x_forwarded_for.split(",")[0]
     else:
         ip = request.META.get("REMOTE_ADDR")
+    if ip:
+        ip = re.match(r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})", ip)[0]
     return ip
 
 

geonode/base/fixtures/regions.json

@@ -4345,10 +4345,10 @@
       "level": 1,
       "lft": 464,
       "tree_id": 90,
-      "bbox_x0": 139.572356,
-      "bbox_x1": -74.531208,
-      "bbox_y0": -56.267241,
-      "bbox_y1": 62.0215969
+      "bbox_x0": 112.921112,
+      "bbox_x1": -108.87291,
+      "bbox_y0": -54.640301,
+      "bbox_y1": 20.41712
     }
   },
   {

geonode/documents/api/serializers.py

@@ -61,3 +61,10 @@ class Meta:
             "doc_url",
             "metadata",
         )
+
+    def to_representation(self, obj):
+        _doc = super(DocumentSerializer, self).to_representation(obj)
+        # better to hide internal server file path
+        _doc.pop("file_path")
+        _doc.pop("doc_file")
+        return _doc

geonode/documents/api/tests.py

@@ -16,7 +16,6 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 #########################################################################
-import os
 import logging
 
 from django.contrib.auth import get_user_model
@@ -139,14 +138,20 @@ def test_creation_should_create_the_doc(self):
         }
         actual = self.client.post(self.url, data=payload, format="json")
         self.assertEqual(201, actual.status_code)
-        cloned_path = actual.json().get("document", {}).get("file_path", "")[0]
         extension = actual.json().get("document", {}).get("extension", "")
-        self.assertTrue(os.path.exists(cloned_path))
         self.assertEqual("xml", extension)
         self.assertTrue(Document.objects.filter(title="New document for testing").exists())
 
-        if cloned_path:
-            os.remove(cloned_path)
+    def test_file_path_and_doc_path_are_not_returned(self):
+        """
+        If file_path and doc_path should not be visible
+        from the GET payload
+        """
+        actual = self.client.get(self.url)
+        self.assertEqual(200, actual.status_code)
+        _doc_payload = actual.json().get("document", {})
+        self.assertFalse("file_path" in _doc_payload)
+        self.assertFalse("doc_path" in _doc_payload)
 
     def test_creation_from_url_should_create_the_doc(self):
         """

geonode/geoserver/management/commands/set_default_gridsets.py

@@ -78,11 +78,11 @@ def handle(self, **options):
                     """
                     curl -v -u admin:geoserver -XPOST \
                         -H "Content-type: text/xml" -d @poi.xml \
-                            "http://localhost:8080/geoserver/gwc/rest/datasets/tiger:poi.xml"
+                            "http://localhost:8080/geoserver/gwc/rest/layers/tiger:poi.xml"
                     """
                     headers = {'Content-type': 'text/xml'}
                     payload = ET.tostring(tree)
-                    r = requests.post(f'{url}gwc/rest/datasets/{layer.typename}.xml',
+                    r = requests.post(f'{url}gwc/rest/layers/{layer.typename}.xml',
                                       headers=headers,
                                       data=payload,
                                       auth=HTTPBasicAuth(user, passwd))

geonode/resource/utils.py

@@ -489,7 +489,7 @@ def metadata_post_save(instance, *args, **kwargs):
                     poly2 = GEOSGeometry(wkt2, srid=int(srid2[0]))
                     poly2.transform(4326)
 
-                    if poly2.intersection(poly1):
+                    if not poly2.intersection(poly1).empty:
                         regions_to_add.append(region)
                     if region.level == 0 and region.parent is None:
                         global_regions.append(region)

geonode/settings.py

@@ -2226,7 +2226,7 @@ def get_geonode_catalogue_service():
         "id": "tiff",
         "label": "GeoTIFF",
         "format": "raster",
-        "ext": ["tiff", "tif"],
+        "ext": ["tiff", "tif", "geotiff", "geotif"],
         "mimeType": ["image/tiff"],
         "optional": ["xml", "sld"],
     },

geonode/tests/test_utils.py

@@ -28,7 +28,7 @@
 from unittest.mock import patch
 from datetime import datetime, timedelta
 
-from django.contrib.gis.geos import Polygon
+from django.contrib.gis.geos import GEOSGeometry, Polygon
 from django.contrib.auth import get_user_model
 from django.core.management import call_command
 
@@ -37,7 +37,7 @@
 from geonode.geoserver.helpers import set_attributes
 from geonode.tests.base import GeoNodeBaseTestSupport
 from geonode.br.management.commands.utils.utils import ignore_time
-from geonode.utils import copy_tree, fixup_shp_columnnames, get_supported_datasets_file_types, unzip_file
+from geonode.utils import copy_tree, fixup_shp_columnnames, get_supported_datasets_file_types, unzip_file, bbox_to_wkt
 from geonode import settings
 
 
@@ -286,3 +286,68 @@ def test_should_replace_the_type_id_if_already_exists(self):
         self.assertEqual(len(supported_keys), prev_count)
         shp_type = [t for t in supported_types if t["id"] == "shp"][0]
         self.assertEqual(shp_type["label"], "Replaced type")
+
+
+class TestRegionsCrossingDateLine(TestCase):
+    def setUp(self):
+        self.test_region_coords = [
+            [112.921111999999994, -108.872910000000005, -54.640301000000001, 20.417120000000001],  # Pacific
+            [174.866196000000002, -178.203156000000007, -21.017119999999998, -12.466220000000000],  # Fiji
+            [158.418335000000013, -150.208359000000002, -11.437030000000000, 4.719560000000000],  # Kiribati
+            [165.883803999999998, -175.987198000000006, -52.618591000000002, -29.209969999999998],  # New Zeland
+        ]
+
+        self.region_across_idl = [
+            112.921111999999994,
+            -108.872910000000005,
+            -54.640301000000001,
+            20.417120000000001,
+        ]  # Pacific
+        self.region_not_across_idl = [
+            -31.266000999999999,
+            39.869301000000000,
+            27.636310999999999,
+            81.008797000000001,
+        ]  # Europe
+
+        self.dataset_crossing = GEOSGeometry(
+            "POLYGON ((-179.30100799101168718 -17.31310259828852693, -170.41740336774466869 -9.63481116511300328, -164.30155495876181249 -19.7237289784715415, \
+            -177.91712988386967709 -30.43762400150689018, -179.30100799101168718 -17.31310259828852693))",
+            srid=4326,
+        )
+        self.dataset_not_crossing = GEOSGeometry(
+            "POLYGON ((-41.86461347546176626 5.43160371103088835, 34.20404118809119609 4.3602142087273279, 9.56208263510924894 -48.8521310723496498,  \
+            -42.22174330956295307 -32.42415870369499942, -41.86461347546176626 5.43160371103088835))",
+            srid=4326,
+        )
+
+    def test_region_across_dateline_do_not_intersect_areas_outside(self):
+        for i, region_coords in enumerate(self.test_region_coords):
+            geographic_bounding_box = bbox_to_wkt(
+                region_coords[0], region_coords[1], region_coords[2], region_coords[3]
+            )
+            _, wkt = geographic_bounding_box.split(";")
+            poly = GEOSGeometry(wkt, srid=4326)
+
+            self.assertFalse(
+                poly.intersection(self.dataset_crossing).empty, f"True intersection not detected for region {i}"
+            )
+            self.assertTrue(poly.intersection(self.dataset_not_crossing).empty, "False intersection detected")
+
+    def test_region_wkt_multipolygon_if_across_idl(self):
+        bbox_across_idl = bbox_to_wkt(
+            self.region_not_across_idl[0],
+            self.region_not_across_idl[1],
+            self.region_not_across_idl[2],
+            self.region_not_across_idl[3],
+        )
+        _, wkt = bbox_across_idl.split(";")
+        poly = GEOSGeometry(wkt, srid=4326)
+        self.assertEqual(poly.geom_type, "Polygon", f"Expexted 'Polygon' type but received {poly.geom_type}")
+
+        bbox_across_idl = bbox_to_wkt(
+            self.region_across_idl[0], self.region_across_idl[1], self.region_across_idl[2], self.region_across_idl[3]
+        )
+        _, wkt = bbox_across_idl.split(";")
+        poly = GEOSGeometry(wkt, srid=4326)
+        self.assertEqual(poly.geom_type, "MultiPolygon", f"Expexted 'MultiPolygon' type but received {poly.geom_type}")

geonode/utils.py

@@ -496,9 +496,44 @@ def bbox_to_wkt(x0, x1, y0, y1, srid="4326", include_srid=True):
     if srid and str(srid).startswith("EPSG:"):
         srid = srid[5:]
     if None not in {x0, x1, y0, y1}:
-        wkt = "POLYGON(({:f} {:f},{:f} {:f},{:f} {:f},{:f} {:f},{:f} {:f}))".format(
-            float(x0), float(y0), float(x0), float(y1), float(x1), float(y1), float(x1), float(y0), float(x0), float(y0)
+        polys = []
+
+        # We assume that if x1 is smaller then x0 we're crossing the date line
+        crossing_idl = x1 < x0
+        if crossing_idl:
+            polys.append(
+                [
+                    (float(x0), float(y0)),
+                    (float(x0), float(y1)),
+                    (180.0, float(y1)),
+                    (180.0, float(y0)),
+                    (float(x0), float(y0)),
+                ]
+            )
+            polys.append(
+                [
+                    (-180.0, float(y0)),
+                    (-180.0, float(y1)),
+                    (float(x1), float(y1)),
+                    (float(x1), float(y0)),
+                    (-180.0, float(y0)),
+                ]
+            )
+        else:
+            polys.append(
+                [
+                    (float(x0), float(y0)),
+                    (float(x0), float(y1)),
+                    (float(x1), float(y1)),
+                    (float(x1), float(y0)),
+                    (float(x0), float(y0)),
+                ]
+            )
+
+        poly_wkts = ",".join(
+            ["(({}))".format(",".join(["{:f} {:f}".format(coords[0], coords[1]) for coords in poly])) for poly in polys]
         )
+        wkt = f"MULTIPOLYGON({poly_wkts})" if len(polys) > 1 else f"POLYGON{poly_wkts}"
         if include_srid:
             wkt = f"SRID={srid};{wkt}"
     else:

requirements.txt

@@ -54,9 +54,9 @@ pyjwt==2.7.0
 
 # geopython dependencies
 pyproj<3.6.0
-OWSLib==0.29.1
+OWSLib==0.29.2
 pycsw==2.6.1
-SQLAlchemy==2.0.13 # required by PyCSW
+SQLAlchemy==2.0.15 # required by PyCSW
 Shapely==1.8.5.post1
 mercantile==1.2.1
 geoip2==4.7.0
@@ -97,7 +97,7 @@ geonode-user-messages==2.0.2
 geonode-announcements==2.0.2
 geonode-django-activity-stream==0.10.0
 gn-arcrest==10.5.5
-geoserver-restconfig==2.0.8
+geoserver-restconfig==2.0.9
 gn-gsimporter==2.0.4
 gisdata==0.5.4
 
@@ -113,7 +113,7 @@ django-storages==1.13.2
 dropbox==11.36.0
 google-cloud-storage==2.9.0
 google-cloud-core==2.3.2
-boto3==1.26.133
+boto3==1.26.137
 
 # Django Caches
 python-memcached<=1.59
@@ -127,7 +127,7 @@ jdcal==1.4.1
 mock<6.0.0
 python-dateutil==2.8.2
 pytz==2023.3
-requests==2.30.0
+requests==2.31.0
 timeout-decorator==0.5.0
 pylibmc==1.6.3
 sherlock==0.4.1
@@ -147,7 +147,7 @@ uWSGI==2.0.21
 gunicorn==20.1.0
 ipython==8.13.2
 docker==6.1.2
-invoke==2.1.1
+invoke==2.1.2
 
 # tests
 coverage==7.2.5
@@ -159,7 +159,7 @@ pytest-bdd==6.1.1
 splinter==0.19.0
 pytest-splinter==3.3.2
 pytest-django==4.5.2
-setuptools>=59.1.1,<67.8.0
+setuptools>=59.1.1,<67.9.0
 pip==23.1.2
 Twisted==22.10.0
 pixelmatch==0.3.0
@@ -171,7 +171,7 @@ webdriver_manager==3.8.6
 
 # Security and audit
 mistune==2.0.5
-wandb==0.15.2
+wandb==0.15.3
 protobuf==3.20.3
 mako==1.2.4
 certifi>=2022.12.7 # not directly required, pinned by Snyk to avoid a vulnerability

scripts/docker/nginx/Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:1.23.3-alpine
+FROM nginx:1-alpine
 
 RUN apk add --no-cache openssl inotify-tools
 

setup.cfg

@@ -80,9 +80,9 @@ install_requires =
 
     # geopython dependencies
     pyproj<3.6.0
-    OWSLib==0.29.1
+    OWSLib==0.29.2
     pycsw==2.6.1
-    SQLAlchemy==2.0.13 # required by PyCSW
+    SQLAlchemy==2.0.15 # required by PyCSW
     Shapely==1.8.5.post1
     mercantile==1.2.1
     geoip2==4.7.0
@@ -115,14 +115,14 @@ install_requires =
 
     # GeoNode org maintained apps.
     django-geonode-mapstore-client>=4.0.5,<5.0.0
-    geonode-importer>=1.0.0
+    geonode-importer>=1.0.2
     geonode-avatar==5.0.8
     geonode-oauth-toolkit==2.2.2
     geonode-user-messages==2.0.2
     geonode-announcements==2.0.2
     geonode-django-activity-stream==0.10.0
     gn-arcrest==10.5.5
-    geoserver-restconfig==2.0.8
+    geoserver-restconfig==2.0.9
     gn-gsimporter==2.0.4
     gisdata==0.5.4
 
@@ -138,7 +138,7 @@ install_requires =
     dropbox==11.36.0
     google-cloud-storage==2.9.0
     google-cloud-core==2.3.2
-    boto3==1.26.133
+    boto3==1.26.137
 
     # Django Caches
     python-memcached<=1.59
@@ -152,7 +152,7 @@ install_requires =
     mock<6.0.0
     python-dateutil==2.8.2
     pytz==2023.3
-    requests==2.30.0
+    requests==2.31.0
     timeout-decorator==0.5.0
     pylibmc==1.6.3
     sherlock==0.4.1
@@ -172,7 +172,7 @@ install_requires =
     gunicorn==20.1.0
     ipython==8.13.2
     docker==6.1.2
-    invoke==2.1.1
+    invoke==2.1.2
 
     # tests
     coverage==7.2.5
@@ -184,7 +184,7 @@ install_requires =
     splinter==0.19.0
     pytest-splinter==3.3.2
     pytest-django==4.5.2
-    setuptools>=59.1.1,<67.8.0
+    setuptools>=59.1.1,<67.9.0
     pip==23.1.2
     Twisted==22.10.0
     pixelmatch==0.3.0
@@ -196,7 +196,7 @@ install_requires =
 
     # Security and audit
     mistune==2.0.5
-    wandb==0.15.2
+    wandb==0.15.3
     protobuf==3.20.3
     mako==1.2.4
     certifi>=2022.12.7 # not directly required, pinned by Snyk to avoid a vulnerability