diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index b831326e88a..2b6933b4633 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -29,6 +29,7 @@ from django.db.models.query import QuerySet from django.http import QueryDict +from deprecated import deprecated from rest_framework import serializers from rest_framework_gis import fields from rest_framework.reverse import reverse, NoReverseMatch @@ -54,8 +55,9 @@ ) from geonode.groups.models import GroupCategory, GroupProfile from geonode.base.api.fields import ComplexDynamicRelationField +from geonode.layers.utils import get_dataset_download_handlers, get_default_dataset_download_handler from geonode.utils import build_absolute_uri -from geonode.security.utils import get_resources_with_perms +from geonode.security.utils import get_resources_with_perms, get_geoapp_subtypes from geonode.resource.models import ExecutionRequest logger = logging.getLogger(__name__) @@ -278,8 +280,12 @@ class DownloadLinkField(DynamicComputedField): def __init__(self, **kwargs): super().__init__(**kwargs) + @deprecated(version="4.2.0", reason="Will be replaced by download_urls") def get_attribute(self, instance): try: + logger.info( + "This field is deprecated, and will be removed in the future GeoNode version. Please refer to download_urls" + ) _instance = instance.get_real_instance() return _instance.download_url if hasattr(_instance, "download_url") else None except Exception as e: @@ -287,6 +293,43 @@ def get_attribute(self, instance): return None +class DownloadArrayLinkField(DynamicComputedField): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_attribute(self, instance): + try: + _instance = instance.get_real_instance() + except Exception as e: + logger.exception(e) + raise e + if _instance.resource_type in ["map"] + get_geoapp_subtypes(): + return [] + elif _instance.resource_type in ["document"]: + return [ + { + "url": _instance.download_url, + "ajax_safe": _instance.download_is_ajax_safe, + } + ] + elif _instance.resource_type in ["dataset"]: + download_urls = [] + # lets get only the default one first to set it + default_handler = get_default_dataset_download_handler() + obj = default_handler(self.context.get("request"), _instance.alternate) + if obj.download_url: + download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": True}) + # then let's prepare the payload with everything + handler_list = get_dataset_download_handlers() + for handler in handler_list: + obj = handler(self.context.get("request"), _instance.alternate) + if obj.download_url: + download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": False}) + return download_urls + else: + return [] + + class FavoriteField(DynamicComputedField): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -479,6 +522,7 @@ def __init__(self, *args, **kwargs): self.fields["is_copyable"] = serializers.BooleanField(read_only=True) self.fields["download_url"] = DownloadLinkField(read_only=True) self.fields["favorite"] = FavoriteField(read_only=True) + self.fields["download_urls"] = DownloadArrayLinkField(read_only=True) metadata = ComplexDynamicRelationField(ExtraMetadataSerializer, embed=False, many=True, deferred=True) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 4feffb7d3c2..961d28da16a 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -23,7 +23,7 @@ import json import logging from django.contrib.contenttypes.models import ContentType -from django.test import override_settings +from django.test import RequestFactory, override_settings import gisdata from PIL import Image @@ -751,12 +751,16 @@ def test_resource_serializer_validation(self): ds = ResourceBase.objects.get(title=title) ds.keywords.add(HierarchicalKeyword.objects.get(slug="a1")) - serialized = ResourceBaseSerializer(ds) + factory = RequestFactory() + rq = factory.get("test") + rq.user = owner + + serialized = ResourceBaseSerializer(ds, context={"request": rq}) json = JSONRenderer().render(serialized.data) stream = BytesIO(json) data = JSONParser().parse(stream) self.assertIsInstance(data, dict) - se = ResourceBaseSerializer(data=data) + se = ResourceBaseSerializer(data=data, context={"request": rq}) self.assertTrue(se.is_valid()) def test_delete_user_with_resource(self): @@ -2065,6 +2069,7 @@ def test_manager_can_edit_map(self): """ REST API must not forbid saving maps and apps to non-admin and non-owners. """ + self.maxDiff = None from geonode.maps.models import Map _map = Map.objects.filter(uuid__isnull=False, owner__username="admin").first() @@ -2120,7 +2125,7 @@ def test_manager_can_edit_map(self): response = self.client.get(resource_service_permissions_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data - self.assertEqual( + self.assertDictEqual( resource_perm_spec, { "users": [ @@ -2161,7 +2166,7 @@ def test_manager_can_edit_map(self): response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data - self.assertEqual( + self.assertDictEqual( resource_perm_spec, { "users": [ @@ -2200,20 +2205,10 @@ def test_manager_can_edit_map(self): response = self.client.get(get_perms_url, format="json") self.assertEqual(response.status_code, 200) resource_perm_spec = response.data - self.assertEqual( + self.assertDictEqual( resource_perm_spec, { "users": [ - { - "id": 1, - "username": "admin", - "first_name": "admin", - "last_name": "", - "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", - "permissions": "owner", - "is_staff": True, - "is_superuser": True, - }, { "id": bobby.id, "username": "bobby", @@ -2224,6 +2219,16 @@ def test_manager_can_edit_map(self): "is_staff": False, "is_superuser": False, }, + { + "id": 1, + "username": "admin", + "first_name": "admin", + "last_name": "", + "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240", + "permissions": "owner", + "is_staff": True, + "is_superuser": True, + }, ], "organizations": [], "groups": [ @@ -2445,6 +2450,59 @@ def test_base_resources_dont_return_download_link_if_map(self): download_url = response.json().get("resource").get("download_url") self.assertIsNone(download_url) + def test_base_resources_return_not_download_links_for_maps(self): + """ + Ensure we can access the Resource Base list. + """ + _map = Map.objects.first() + # From resource base API + url = reverse("base-resources-detail", args=[_map.id]) + response = self.client.get(url, format="json") + download_url = response.json().get("resource").get("download_urls", None) + self.assertListEqual([], download_url) + + # from maps api + url = reverse("maps-detail", args=[_map.id]) + download_url = response.json().get("resource").get("download_urls") + self.assertListEqual([], download_url) + + def test_base_resources_return_download_links_for_documents(self): + """ + Ensure we can access the Resource Base list. + """ + doc = Document.objects.first() + expected_payload = [{"url": build_absolute_uri(doc.download_url), "ajax_safe": doc.download_is_ajax_safe}] + # From resource base API + url = reverse("base-resources-detail", args=[doc.id]) + response = self.client.get(url, format="json") + download_url = response.json().get("resource").get("download_urls") + self.assertListEqual(expected_payload, download_url) + + # from documents api + url = reverse("documents-detail", args=[doc.id]) + download_url = response.json().get("resource").get("download_urls") + self.assertListEqual(expected_payload, download_url) + + def test_base_resources_return_download_links_for_datasets(self): + """ + Ensure we can access the Resource Base list. + """ + _dataset = Dataset.objects.first() + expected_payload = [ + {"url": reverse("dataset_download", args=[_dataset.alternate]), "ajax_safe": True, "default": True} + ] + + # From resource base API + url = reverse("base-resources-detail", args=[_dataset.id]) + response = self.client.get(url, format="json") + download_url = response.json().get("resource").get("download_urls") + self.assertEqual(expected_payload, download_url) + + # from dataset api + url = reverse("datasets-detail", args=[_dataset.id]) + download_url = response.json().get("resource").get("download_urls") + self.assertEqual(expected_payload, download_url) + class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport): def setUp(self): diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index ae81b395883..fbbd2faec29 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -378,7 +378,7 @@ def _filtered(self, request, filter): description="API endpoint allowing to retrieve the approved Resources.", ) @action(detail=False, methods=["get"]) - def approved(self, request): + def approved(self, request, *args, **kwargs): return self._filtered(request, {"is_approved": True}) @extend_schema( @@ -387,7 +387,7 @@ def approved(self, request): description="API endpoint allowing to retrieve the published Resources.", ) @action(detail=False, methods=["get"]) - def published(self, request): + def published(self, request, *args, **kwargs): return self._filtered(request, {"is_published": True}) @extend_schema( @@ -396,7 +396,7 @@ def published(self, request): description="API endpoint allowing to retrieve the featured Resources.", ) @action(detail=False, methods=["get"]) - def featured(self, request): + def featured(self, request, *args, **kwargs): return self._filtered(request, {"featured": True}) @extend_schema( @@ -411,7 +411,7 @@ def featured(self, request): IsAuthenticated, ], ) - def favorites(self, request, pk=None): + def favorites(self, request, pk=None, *args, **kwargs): paginator = GeoNodeApiPagination() paginator.page_size = request.GET.get("page_size", 10) favorites = Favorite.objects.favorites_for_user(user=request.user) @@ -425,7 +425,7 @@ def favorites(self, request, pk=None): description="API endpoint allowing to retrieve the favorite Resources.", ) @action(detail=True, methods=["post", "delete"], permission_classes=[IsAuthenticated]) - def favorite(self, request, pk=None): + def favorite(self, request, pk=None, *args, **kwargs): resource = self.get_object() user = request.user @@ -476,7 +476,7 @@ def favorite(self, request, pk=None): """, ) @action(detail=False, methods=["get"]) - def resource_types(self, request): + def resource_types(self, request, *args, **kwargs): def _to_compact_perms_list( allowed_perms: dict, resource_type: str, resource_subtype: str, compact_perms_labels: dict = {} ) -> list: @@ -583,7 +583,7 @@ def _to_compact_perms_list( methods=["get", "put", "patch", "delete"], permission_classes=[IsAuthenticated], ) - def resource_service_permissions(self, request, pk): + def resource_service_permissions(self, request, pk, *args, **kwargs): """Instructs the Async dispatcher to execute a 'DELETE' or 'UPDATE' on the permissions of a valid 'uuid' - GET input_params: { @@ -720,7 +720,7 @@ def resource_service_permissions(self, request, pk): methods=["post"], permission_classes=[IsAuthenticated, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], ) - def set_thumbnail_from_bbox(self, request, resource_id): + def set_thumbnail_from_bbox(self, request, resource_id, *args, **kwargs): import traceback from django.utils.datastructures import MultiValueDictKeyError @@ -778,7 +778,7 @@ def set_thumbnail_from_bbox(self, request, resource_id): methods=["post"], permission_classes=[IsAuthenticated], ) - def resource_service_ingest(self, request, resource_type: str = None): + def resource_service_ingest(self, request, resource_type: str = None, *args, **kwargs): """Instructs the Async dispatcher to execute a 'INGEST' operation - POST input_params: { @@ -879,7 +879,7 @@ def resource_service_ingest(self, request, resource_type: str = None): methods=["post"], permission_classes=[IsAuthenticated, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})], ) - def resource_service_create(self, request, resource_type: str = None): + def resource_service_create(self, request, resource_type: str = None, *args, **kwargs): """Instructs the Async dispatcher to execute a 'CREATE' operation **WARNING**: This will create an empty dataset; if you need to upload a resource to GeoNode, consider using the endpoint "ingest" instead @@ -981,7 +981,7 @@ def resource_service_create(self, request, resource_type: str = None): methods=["delete"], permission_classes=[IsAuthenticated, UserHasPerms], ) - def resource_service_delete(self, request, pk): + def resource_service_delete(self, request, pk, *args, **kwargs): """Instructs the Async dispatcher to execute a 'DELETE' operation over a valid 'uuid' - DELETE input_params: { @@ -1065,7 +1065,7 @@ def resource_service_delete(self, request, pk): methods=["put"], permission_classes=[IsAuthenticated, UserHasPerms], ) - def resource_service_update(self, request, pk): + def resource_service_update(self, request, pk, *args, **kwargs): """Instructs the Async dispatcher to execute a 'UPDATE' operation over a valid 'uuid' - PUT input_params: { @@ -1195,7 +1195,7 @@ def resource_service_update(self, request, pk): ), ], ) - def resource_service_copy(self, request, pk): + def resource_service_copy(self, request, pk, *args, **kwargs): """Instructs the Async dispatcher to execute a 'COPY' operation over a valid 'pk' - PUT input_params: { @@ -1300,7 +1300,7 @@ def resource_service_copy(self, request, pk): UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}}), ], ) - def ratings(self, request, pk): + def ratings(self, request, pk, *args, **kwargs): resource = get_object_or_404(ResourceBase, pk=pk) resource = resource.get_real_instance() ct = ContentType.objects.get_for_model(resource) @@ -1337,7 +1337,7 @@ def ratings(self, request, pk): permission_classes=[IsAuthenticated, UserHasPerms], parser_classes=[JSONParser, MultiPartParser], ) - def set_thumbnail(self, request, pk): + def set_thumbnail(self, request, pk, *args, **kwargs): resource = get_object_or_404(ResourceBase, pk=pk) if not request.data.get("file"): @@ -1397,7 +1397,7 @@ def set_thumbnail(self, request, pk): url_path=r"extra_metadata", # noqa url_name="extra-metadata", ) - def extra_metadata(self, request, pk): + def extra_metadata(self, request, pk, *args, **kwargs): _obj = get_object_or_404(ResourceBase, pk=pk) if request.method == "GET": @@ -1488,7 +1488,7 @@ def _get_request_params(self, request, encode=False): url_path=r"linked_resources", # noqa url_name="linked_resources", ) - def linked_resources(self, request, pk): + def linked_resources(self, request, pk, *args, **kwargs): try: """ To let the API be able to filter the linked result, we cannot rely on the DynamicFilterBackend diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index 720cb7639ba..dd15a23e363 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -145,7 +145,7 @@ def perform_create(self, serializer): description="API endpoint allowing to retrieve the DocumentResourceLink(s).", ) @action(detail=True, methods=["get"]) - def linked_resources(self, request, pk=None): + def linked_resources(self, request, pk=None, *args, **kwargs): document = self.get_object() resources_id = document.links.all().values("object_id") resources = ResourceBase.objects.filter(id__in=resources_id) diff --git a/geonode/documents/models.py b/geonode/documents/models.py index 402194f9908..5f5f41ab218 100644 --- a/geonode/documents/models.py +++ b/geonode/documents/models.py @@ -103,6 +103,14 @@ def href(self): elif self.files: return urljoin(settings.SITEURL, reverse("document_link", args=(self.id,))) + @property + def is_local(self): + return False if self.doc_url else True + + @property + def download_is_ajax_safe(self): + return self.is_local + @property def is_file(self): return self.files and self.extension diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 8bf91a88529..f58a71f4734 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -56,7 +56,7 @@ from geonode.documents.enumerations import DOCUMENT_TYPE_MAP from geonode.documents.models import Document, DocumentResourceLink -from geonode.base.populate_test_data import all_public, create_models, remove_models +from geonode.base.populate_test_data import all_public, create_models, create_single_doc, remove_models from geonode.upload.api.exceptions import FileUploadLimitException from .forms import DocumentCreateForm @@ -153,6 +153,26 @@ def test_remote_document_is_marked_remote(self): d = Document.objects.get(title="A remote document through form is remote") self.assertEqual(d.sourcetype, SOURCE_TYPE_REMOTE) + def test_download_is_not_ajax_safe(self): + """Remote document is mark as not safe.""" + self.client.login(username="admin", password="admin") + form_data = { + "title": "A remote document through form is remote", + "doc_url": "https://development.demo.geonode.org/static/mapstore/img/geonode-logo.svg", + } + + response = self.client.post(reverse("document_upload"), data=form_data) + + self.assertEqual(response.status_code, 302) + + d = Document.objects.get(title="A remote document through form is remote") + self.assertFalse(d.download_is_ajax_safe) + + def test_download_is_ajax_safe(self): + """Remote document is mark as not safe.""" + d = create_single_doc("example_doc_name") + self.assertTrue(d.download_is_ajax_safe) + def test_create_document_url(self): """Tests creating an external document instead of a file.""" diff --git a/geonode/geoserver/tests/test_server.py b/geonode/geoserver/tests/test_server.py index 1fbafbff07f..42d7ec06bb9 100644 --- a/geonode/geoserver/tests/test_server.py +++ b/geonode/geoserver/tests/test_server.py @@ -48,6 +48,7 @@ from geonode.layers.populate_datasets_data import create_dataset_data from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset from geonode.geoserver.helpers import gs_catalog, get_sld_for, extract_name_from_sld +from geonode.catalogue.models import catalogue_post_save import logging @@ -1192,8 +1193,19 @@ def test_set_resources_links(self): with self.settings(UPDATE_RESOURCE_LINKS_AT_MIGRATE=True, ASYNC_SIGNALS=False): # Links _def_link_types = ["original", "metadata"] - _links = Link.objects.filter(link_type__in=_def_link_types) # Check 'original' and 'metadata' links exist + Link.objects.update_or_create( + resource=Dataset.objects.first(), + url="https://custom_dowonload_url.com", + defaults=dict( + extension="zip", + name="Original Dataset", + mime="application/octet-stream", + link_type="original", + ), + ) + _links = Link.objects.filter(link_type__in=_def_link_types) + self.assertIsNotNone(_links, "No 'original' and 'metadata' links have been found") # Delete all 'original' and 'metadata' links @@ -1233,6 +1245,18 @@ def test_set_resources_links(self): for _lyr in _post_migrate_datasets: # Check original links in csw_anytext + # by default is not created anymore, we need to create one + Link.objects.update_or_create( + resource=_lyr, + url="https://custom_dowonload_url.com", + defaults=dict( + extension="zip", + name="Original Dataset", + mime="application/octet-stream", + link_type="original", + ), + ) + _post_migrate_links_orig = Link.objects.filter( resource=_lyr.resourcebase_ptr, resource_id=_lyr.resourcebase_ptr.id, link_type="original" ) @@ -1240,6 +1264,9 @@ def test_set_resources_links(self): _post_migrate_links_orig.exists(), f"No 'original' links has been found for the layer '{_lyr.alternate}'", ) + # needed to update the csw_anytext field with the new link created + catalogue_post_save(instance=_lyr, sender=_lyr.__class__) + _lyr.refresh_from_db() for _link_orig in _post_migrate_links_orig: self.assertIn( _link_orig.url, diff --git a/geonode/layers/api/tests.py b/geonode/layers/api/tests.py index 7361e5c744a..88ba047ed03 100644 --- a/geonode/layers/api/tests.py +++ b/geonode/layers/api/tests.py @@ -30,6 +30,7 @@ from guardian.shortcuts import assign_perm, get_anonymous_user from geonode.geoserver.createlayer.utils import create_dataset +from geonode.base.models import Link from geonode.base.populate_test_data import create_models, create_single_dataset from geonode.layers.models import Attribute, Dataset from geonode.maps.models import Map, MapLayer @@ -415,3 +416,26 @@ def test_valid_metadata_file(self): put_data = {"metadata_file": f} response = self.client.put(url, data=put_data) self.assertEqual(200, response.status_code) + + def test_download_api(self): + dataset = create_single_dataset("test_dataset") + url = reverse("datasets-detail", kwargs={"pk": dataset.pk}) + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + data = response.json()["dataset"] + download_url_data = data["download_urls"][0] + download_url = reverse("dataset_download", args=[dataset.alternate]) + self.assertEqual(download_url_data["default"], True) + self.assertEqual(download_url_data["ajax_safe"], True) + self.assertEqual(download_url_data["url"], download_url) + + link = Link(link_type="original", url="https://myoriginal.org", resource=dataset) + link.save() + + response = self.client.get(url) + data = response.json()["dataset"] + download_url_data = data["download_urls"][0] + download_url = reverse("dataset_download", args=[dataset.alternate]) + self.assertEqual(download_url_data["default"], True) + self.assertEqual(download_url_data["ajax_safe"], False) + self.assertEqual(download_url_data["url"], "https://myoriginal.org") diff --git a/geonode/layers/api/views.py b/geonode/layers/api/views.py index 3ad985c7781..218216a17bd 100644 --- a/geonode/layers/api/views.py +++ b/geonode/layers/api/views.py @@ -100,7 +100,7 @@ def get_serializer_class(self): UserHasPerms(perms_dict={"default": {"PUT": ["base.change_resourcebase_metadata"]}}), ], ) - def metadata(self, request, pk=None): + def metadata(self, request, pk=None, *args, **kwargs): """ Endpoint to upload ISO metadata Usage Example: @@ -165,7 +165,7 @@ def metadata(self, request, pk=None): description="API endpoint allowing to retrieve the MapLayers list.", ) @action(detail=True, methods=["get"]) - def maplayers(self, request, pk=None): + def maplayers(self, request, pk=None, *args, **kwargs): dataset = self.get_object() resources = dataset.maplayers return Response(SimpleMapLayerSerializer(many=True).to_representation(resources)) @@ -176,7 +176,7 @@ def maplayers(self, request, pk=None): description="API endpoint allowing to retrieve maps using the dataset.", ) @action(detail=True, methods=["get"]) - def maps(self, request, pk=None): + def maps(self, request, pk=None, *args, **kwargs): dataset = self.get_object() resources = dataset.maps return Response(SimpleMapSerializer(many=True).to_representation(resources)) @@ -208,7 +208,7 @@ def maps(self, request, pk=None): methods=["patch"], serializer_class=DatasetReplaceAppendSerializer, ) - def replace(self, request, dataset_id=None): + def replace(self, request, dataset_id=None, *args, **kwargs): """ Edpoint for replace data to an existing layer """ @@ -241,7 +241,7 @@ def replace(self, request, dataset_id=None): methods=["patch"], serializer_class=DatasetReplaceAppendSerializer, ) - def append(self, request, dataset_id=None): + def append(self, request, dataset_id=None, *args, **kwargs): """ Edpoint for replace data to an existing layer """ diff --git a/geonode/resource/download_handler.py b/geonode/layers/download_handler.py similarity index 73% rename from geonode/resource/download_handler.py rename to geonode/layers/download_handler.py index ae21e4e23a9..d969a0d3703 100644 --- a/geonode/resource/download_handler.py +++ b/geonode/layers/download_handler.py @@ -31,10 +31,10 @@ from geonode.proxy.views import fetch_response_headers from geonode.utils import HttpClient -logger = logging.getLogger("geonode.resource.download_handler") +logger = logging.getLogger("geonode.layers.download_handler") -class DownloadHandler: +class DatasetDownloadHandler: def __str__(self): return f"{self.__module__}.{self.__class__.__name__}" @@ -44,6 +44,7 @@ def __repr__(self): def __init__(self, request, resource_name) -> None: self.request = request self.resource_name = resource_name + self._resource = None def get_download_response(self): """ @@ -51,27 +52,61 @@ def get_download_response(self): that allow the resource download """ resource = self.get_resource() + if not resource: + raise Http404("Resource requested is not available") response = self.process_dowload(resource) return response + @property + def is_link_resource(self): + resource = self.get_resource() + return resource.link_set.filter(resource=resource, link_type="original").exists() + + @property + def is_ajax_safe(self): + """ + AJAX is safe to be used for WPS downloads. In case of a link set in a Link entry we cannot assume it, + since it could point to an external (non CORS enabled) URL + """ + return settings.USE_GEOSERVER and not self.is_link_resource + + @property + def download_url(self): + resource = self.get_resource() + if not resource: + return None + if resource.subtype not in ["vector", "raster", "vector_time"]: + logger.info("Download URL is available only for datasets that have been harvested and copied locally") + return None + + if self.is_link_resource: + return resource.link_set.filter(resource=resource.get_self_resource(), link_type="original").first().url + + return reverse("dataset_download", args=[resource.alternate]) + def get_resource(self): """ Returnt the object needed """ - try: - return _resolve_dataset( - self.request, - self.resource_name, - "base.download_resourcebase", - _("You do not have permissions for this dataset."), - ) - except Exception as e: - raise Http404(Exception(_("Not found"), e)) - - def process_dowload(self, resource): + if not self._resource: + try: + self._resource = _resolve_dataset( + self.request, + self.resource_name, + "base.download_resourcebase", + _("You do not have download permissions for this dataset."), + ) + except Exception as e: + logger.exception(e) + + return self._resource + + def process_dowload(self, resource=None): """ Generate the response object """ + if not resource: + resource = self.get_resource() if not settings.USE_GEOSERVER: # if GeoServer is not used, we redirect to the proxy download return HttpResponseRedirect(reverse("download", args=[resource.id])) diff --git a/geonode/layers/models.py b/geonode/layers/models.py index 0d6c425e8e0..05df8956868 100644 --- a/geonode/layers/models.py +++ b/geonode/layers/models.py @@ -330,8 +330,6 @@ def download_url(self): if self.subtype not in ["vector", "raster", "vector_time"]: logger.info("Download URL is available only for datasets that have been harvested and copied locally") return None - if self.link_set.filter(resource=self.get_self_resource(), link_type="original").exists(): - return self.link_set.filter(resource=self.get_self_resource(), link_type="original").first().url return build_absolute_uri(reverse("dataset_download", args=(self.alternate,))) @property diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index 2deb3ca0a21..ec08069bac6 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -41,6 +41,7 @@ from django.contrib.gis.geos import Polygon from django.db.models import Count from django.contrib.auth import get_user_model +from django.http import HttpResponse from django.conf import settings from django.test.utils import override_settings @@ -48,6 +49,7 @@ from geonode.geoserver.createlayer.utils import create_dataset from geonode.layers import utils +from geonode.layers.utils import clear_dataset_download_handlers from geonode.base import enumerations from geonode.layers import DatasetAppConfig from geonode.layers.admin import DatasetAdmin @@ -81,6 +83,7 @@ ) from geonode.base.populate_test_data import all_public, create_models, remove_models, create_single_dataset +from geonode.layers.download_handler import DatasetDownloadHandler logger = logging.getLogger(__name__) @@ -323,6 +326,7 @@ def test_dataset_styles(self): def test_dataset_links(self): lyr = Dataset.objects.filter(subtype="vector").first() self.assertEqual(lyr.subtype, "vector") + if check_ogc_backend(geoserver.BACKEND_PACKAGE): links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="metadata") self.assertIsNotNone(links) @@ -373,11 +377,6 @@ def test_dataset_links(self): links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="image") self.assertIsNotNone(links) - Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="original").update( - url="http://google.com/test" - ) - self.assertEqual(lyr.download_url, "http://google.com/test") - def test_get_valid_user(self): # Verify it accepts an admin user adminuser = get_user_model().objects.get(is_superuser=True) @@ -1210,7 +1209,7 @@ def test_dataset_download_invalid_wps_format(self): self.assertEqual(500, response.status_code) self.assertDictEqual({"error": "The format provided is not valid for the selected resource"}, response.json()) - @patch("geonode.resource.download_handler.HttpClient.request") + @patch("geonode.layers.download_handler.HttpClient.request") def test_dataset_download_call_the_catalog_raise_error_for_no_200(self, mocked_catalog): _response = MagicMock(status_code=500, content="foo-bar") mocked_catalog.return_value = _response, "foo-bar" @@ -1234,7 +1233,7 @@ def test_dataset_download_call_the_catalog_raise_error_for_error_content(self): # if settings.USE_GEOSERVER is false, the URL must be redirected self.client.login(username="admin", password="admin") dataset = Dataset.objects.first() - with patch("geonode.resource.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, content url = reverse("dataset_download", args=[dataset.alternate]) response = self.client.get(url) @@ -1247,7 +1246,7 @@ def test_dataset_download_call_the_catalog_works(self): self.client.login(username="admin", password="admin") dataset = Dataset.objects.first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") - with patch("geonode.resource.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) @@ -1266,21 +1265,21 @@ def test_dataset_download_call_the_catalog_work_anonymous(self): _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa dataset = Dataset.objects.first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") - with patch("geonode.resource.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) self.assertTrue(response.status_code == 200) @override_settings(USE_GEOSERVER=True) - @patch("geonode.resource.download_handler.get_template") + @patch("geonode.layers.download_handler.get_template") def test_dataset_download_call_the_catalog_work_for_raster(self, pathed_template): # if settings.USE_GEOSERVER is false, the URL must be redirected _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa dataset = Dataset.objects.filter(subtype="raster").first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") Dataset.objects.filter(alternate=layer.alternate).update(subtype="raster") - with patch("geonode.resource.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) @@ -1293,13 +1292,13 @@ def test_dataset_download_call_the_catalog_work_for_raster(self, pathed_template ) @override_settings(USE_GEOSERVER=True) - @patch("geonode.resource.download_handler.get_template") + @patch("geonode.layers.download_handler.get_template") def test_dataset_download_call_the_catalog_work_for_vector(self, pathed_template): # if settings.USE_GEOSERVER is false, the URL must be redirected _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa dataset = Dataset.objects.filter(subtype="vector").first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") - with patch("geonode.resource.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) @@ -2219,3 +2218,54 @@ def _assert_perms(self, expected_perms, dataset, username, assertion=True): self.assertSetEqual(expected_perms, actual) else: self.assertFalse(username in [user.username for user in perms["users"]]) + + +class TestDatasetDownloadHandler(GeoNodeBaseTestSupport): + def setUp(self): + user = get_user_model().objects.first() + request = RequestFactory().get("http://test_url.com") + request.user = user + self.dataset = create_single_dataset("test_dataset_for_download") + self.sut = DatasetDownloadHandler(request, self.dataset.alternate) + + def test_download_url_without_original_link(self): + expected_url = reverse("dataset_download", args=[self.dataset.alternate]) + self.assertEqual(expected_url, self.sut.download_url) + + def test_download_url_with_original_link(self): + Link.objects.update_or_create( + resource=self.dataset.resourcebase_ptr, + url="https://custom_dowonload_url.com", + defaults=dict( + extension="zip", + name="Original Dataset", + mime="application/octet-stream", + link_type="original", + ), + ) + expected_url = "https://custom_dowonload_url.com" + self.assertEqual(expected_url, self.sut.download_url) + + def test_get_resource_exists(self): + self.assertIsNotNone(self.sut.get_resource()) + + def test_process_dowload(self): + response = self.sut.get_download_response() + self.assertIsNotNone(response) + + +class DummyDownloadHandler(DatasetDownloadHandler): + def get_download_response(self): + return HttpResponse(content=b"abcsfd2") + + +class TestCustomDownloadHandler(GeoNodeBaseTestSupport): + @override_settings(DEFAULT_DATASET_DOWNLOAD_HANDLER="geonode.layers.tests.DummyDownloadHandler") + def test_download_custom_handler(self): + clear_dataset_download_handlers() + dataset = create_single_dataset("test_custom_download_dataset") + url = reverse("dataset_download", args=[dataset.alternate]) + self.client.login(username="admin", password="admin") + response = self.client.get(url) + self.assertTrue(response.status_code == 200) + self.assertEqual(response.content, b"abcsfd2") diff --git a/geonode/layers/utils.py b/geonode/layers/utils.py index 57b53fc2092..164d07b8562 100644 --- a/geonode/layers/utils.py +++ b/geonode/layers/utils.py @@ -591,3 +591,37 @@ def is_sld_upload_only(request): def mdata_search_by_type(request, filetype): files = list({v.name for k, v in request.FILES.items()}) return len(files) == 1 and all([filetype in f for f in files]) + + +default_dataset_download_handler = None +dataset_download_handler_list = [] + + +def get_dataset_download_handlers(): + if not dataset_download_handler_list and getattr(settings, "DATASET_DOWNLOAD_HANDLERS", None): + dataset_download_handler_list.append(import_string(settings.DATASET_DOWNLOAD_HANDLERS[0])) + + return dataset_download_handler_list + + +def get_default_dataset_download_handler(): + global default_dataset_download_handler + if not default_dataset_download_handler and getattr(settings, "DEFAULT_DATASET_DOWNLOAD_HANDLER", None): + default_dataset_download_handler = import_string(settings.DEFAULT_DATASET_DOWNLOAD_HANDLER) + + return default_dataset_download_handler + + +def set_default_dataset_download_handler(handler): + global default_dataset_download_handler + handler_module = import_string(handler) + if handler_module not in dataset_download_handler_list: + dataset_download_handler_list.append(handler_module) + + default_dataset_download_handler = handler_module + + +def clear_dataset_download_handlers(): + global default_dataset_download_handler + dataset_download_handler_list.clear() + default_dataset_download_handler = None diff --git a/geonode/layers/views.py b/geonode/layers/views.py index d9b08da7f97..3aa3bb91ff8 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -59,7 +59,12 @@ from geonode.decorators import check_keyword_write_perms from geonode.layers.forms import DatasetForm, DatasetTimeSerieForm, LayerAttributeForm, NewLayerUploadForm from geonode.layers.models import Dataset, Attribute -from geonode.layers.utils import is_sld_upload_only, is_xml_upload_only, validate_input_source +from geonode.layers.utils import ( + is_sld_upload_only, + is_xml_upload_only, + get_default_dataset_download_handler, + validate_input_source, +) from geonode.services.models import Service from geonode.base import register_event from geonode.monitoring.models import EventType @@ -69,7 +74,6 @@ from geonode.utils import check_ogc_backend, llbbox_to_mercator, resolve_object, mkdtemp from geonode.geoserver.helpers import ogc_server_settings, select_relevant_files, write_uploaded_files_to_disk from geonode.geoserver.security import set_geowebcache_invalidate_cache -from django.utils.module_loading import import_string if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.helpers import gs_catalog @@ -733,8 +737,8 @@ def dataset_metadata_advanced(request, layername): @csrf_exempt def dataset_download(request, layername): - DownloadHandler = import_string(settings.DATASET_DOWNLOAD_HANDLER) - return DownloadHandler(request, layername).get_download_response() + handler = get_default_dataset_download_handler() + return handler(request, layername).get_download_response() @login_required diff --git a/geonode/maps/api/views.py b/geonode/maps/api/views.py index ae127f8fffb..aec88e47fbe 100644 --- a/geonode/maps/api/views.py +++ b/geonode/maps/api/views.py @@ -96,7 +96,7 @@ def create(self, request, *args, **kwargs): description="API endpoint allowing to retrieve the MapLayers list.", ) @action(detail=True, methods=["get"]) - def maplayers(self, request, pk=None): + def maplayers(self, request, pk=None, *args, **kwargs): map = self.get_object() resources = map.maplayers return Response(MapLayerSerializer(embed=True, many=True).to_representation(resources)) @@ -107,7 +107,7 @@ def maplayers(self, request, pk=None): description="API endpoint allowing to retrieve the local MapLayers.", ) @action(detail=True, methods=["get"]) - def datasets(self, request, pk=None): + def datasets(self, request, pk=None, *args, **kwargs): map = self.get_object() resources = map.datasets return Response(DatasetSerializer(embed=True, many=True).to_representation(resources)) diff --git a/geonode/security/permissions.py b/geonode/security/permissions.py index 0c3ddac2147..4d8b2e0bf1f 100644 --- a/geonode/security/permissions.py +++ b/geonode/security/permissions.py @@ -571,9 +571,9 @@ def compact(self): } ) - json["users"] = user_perms - json["organizations"] = organization_perms - json["groups"] = group_perms + json["users"] = sorted(user_perms, key=lambda x: x["id"], reverse=True) + json["organizations"] = sorted(organization_perms, key=lambda x: x["id"], reverse=True) + json["groups"] = sorted(group_perms, key=lambda x: x["id"], reverse=True) return json.copy() diff --git a/geonode/security/tests.py b/geonode/security/tests.py index 52b43a38551..f80a0ef8a52 100644 --- a/geonode/security/tests.py +++ b/geonode/security/tests.py @@ -2480,7 +2480,9 @@ def test_user_with_view_perms(self): # setting the view permissions url = reverse(_case["url"], kwargs={"pk": _case["resource"].pk}) - _case["resource"].set_permissions({"users": {self.marty.username: ["base.view_resourcebase"]}}) + _case["resource"].set_permissions( + {"users": {self.marty.username: ["base.view_resourcebase", "base.download_resourcebase"]}} + ) # calling the api self.client.force_login(self.marty) result = self.client.get(url) diff --git a/geonode/settings.py b/geonode/settings.py index 48194eaacf9..eae71d843fa 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -1343,9 +1343,6 @@ "Zipped All Files", ] - -DISPLAY_ORIGINAL_DATASET_LINK = ast.literal_eval(os.getenv("DISPLAY_ORIGINAL_DATASET_LINK", "True")) - ACCOUNT_NOTIFY_ON_PASSWORD_CHANGE = ast.literal_eval(os.getenv("ACCOUNT_NOTIFY_ON_PASSWORD_CHANGE", "False")) TASTYPIE_DEFAULT_FORMATS = ["json"] @@ -2361,4 +2358,6 @@ def get_geonode_catalogue_service(): {"class": "geonode.facets.providers.thesaurus.ThesaurusFacetProvider", "config": {"type": "select"}}, ] -DATASET_DOWNLOAD_HANDLER = os.getenv("DATASET_DOWNLOAD_HANDLER", "geonode.resource.download_handler.DownloadHandler") +DEFAULT_DATASET_DOWNLOAD_HANDLER = "geonode.layers.download_handler.DatasetDownloadHandler" + +DATASET_DOWNLOAD_HANDLERS = ast.literal_eval(os.getenv("DATASET_DOWNLOAD_HANDLERS", "[]")) diff --git a/geonode/tests/smoke.py b/geonode/tests/smoke.py index 8b094310722..1317fd8cc36 100644 --- a/geonode/tests/smoke.py +++ b/geonode/tests/smoke.py @@ -19,9 +19,7 @@ from unittest import TestCase -from django.http import HttpResponse from geonode.base.populate_test_data import create_single_dataset -from geonode.resource.download_handler import DownloadHandler from geonode.resource.utils import call_storers from geonode.tests.base import GeoNodeBaseTestSupport @@ -336,21 +334,3 @@ def dummy_metadata_storer2(dataset, custom): if custom.get("second-stage", None): for key, value in custom["second-stage"].items(): setattr(dataset, key, value) - - -class DummyDownloadManager(DownloadHandler): - def get_download_response(self): - return HttpResponse(content=b"abcsfd2") - - -class TestDownloadManager(GeoNodeBaseTestSupport): - def setUp(self): - self.sut = DownloadHandler - - @override_settings(DATASET_DOWNLOAD_HANDLER="geonode.tests.smoke.DummyDownloadManager") - def test_download_handler(self): - dataset = create_single_dataset("test_dataset") - url = reverse("dataset_download", args=[dataset.alternate]) - response = self.client.get(url) - self.assertTrue(response.status_code == 200) - self.assertEqual(response.content, b"abcsfd2") diff --git a/geonode/utils.py b/geonode/utils.py index 694962132f2..26feb074218 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -1411,10 +1411,7 @@ def get_legend_url( def set_resource_default_links(instance, layer, prune=False, **kwargs): from geonode.base.models import Link - from django.urls import reverse from django.utils.translation import ugettext - from geonode.layers.models import Dataset - from geonode.documents.models import Document # Prune old links if prune: @@ -1481,32 +1478,6 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs): logger.exception(e) bbox = instance.bbox_string - # Create Raw Data download link - if settings.DISPLAY_ORIGINAL_DATASET_LINK: - logger.debug(" -- Resource Links[Create Raw Data download link]...") - if isinstance(instance, Dataset): - download_url = build_absolute_uri(reverse("dataset_download", args=(instance.alternate,))) - elif isinstance(instance, Document): - download_url = build_absolute_uri(reverse("document_download", args=(instance.id,))) - else: - download_url = None - - while Link.objects.filter(resource=instance.resourcebase_ptr, link_type="original").exists(): - Link.objects.filter(resource=instance.resourcebase_ptr, link_type="original").delete() - Link.objects.update_or_create( - resource=instance.resourcebase_ptr, - url=download_url, - defaults=dict( - extension="zip", - name="Original Dataset", - mime="application/octet-stream", - link_type="original", - ), - ) - logger.debug(" -- Resource Links[Create Raw Data download link]...done!") - else: - Link.objects.filter(resource=instance.resourcebase_ptr, name="Original Dataset").delete() - # Set download links for WMS, WCS or WFS and KML logger.debug(" -- Resource Links[Set download links for WMS, WCS or WFS and KML]...") instance_ows_url = f"{instance.ows_url}?" if instance.ows_url else f"{ogc_server_settings.public_url}ows?"