From 6056f5feda1634a7513003d5d52d5d9c9ee298e8 Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Tue, 24 Sep 2024 13:04:34 +0200 Subject: [PATCH] [Fixes #12326] Assets: implement migration for old uploaded files (#12411) * [Fixes #12124] GNIP 100: Assets (#12335) * [Fixes #12124] GNIP 100: Assets --------- Co-authored-by: etj * [Fixes #12226] Directory assets (#12337) [Fixes #12226] Directory assets --------- Co-authored-by: etj * [Fixes #12341] Asset download handler and link generator (#12343) * [Fixes #12341] Download handler fix * [Fixes #12341] Assets: link generation (#12342) --------- Co-authored-by: Emanuele Tajariol * [Fixes #12326] Assets: implement migration for old uploaded files * [Fixes #12326] Assets: implement migration for old uploaded files * [Fixes #12326] Assets: implement migration for old uploaded files * [Fixes #12326] rollback requirements --------- Co-authored-by: etj Co-authored-by: Emanuele Tajariol --- geonode/assets/management/__init__.py | 0 .../assets/management/commands/__init__.py | 0 .../commands/migrate_file_to_assets.py | 140 ++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 geonode/assets/management/__init__.py create mode 100644 geonode/assets/management/commands/__init__.py create mode 100644 geonode/assets/management/commands/migrate_file_to_assets.py diff --git a/geonode/assets/management/__init__.py b/geonode/assets/management/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/assets/management/commands/__init__.py b/geonode/assets/management/commands/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/assets/management/commands/migrate_file_to_assets.py b/geonode/assets/management/commands/migrate_file_to_assets.py new file mode 100644 index 00000000000..1b3297c18f0 --- /dev/null +++ b/geonode/assets/management/commands/migrate_file_to_assets.py @@ -0,0 +1,140 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import sys +import shutil +import logging +from django.conf import settings +from geonode.assets.local import LocalAssetHandler +from geonode.assets.models import LocalAsset +from geonode.base.management.commands.helpers import confirm +from django.core.management.base import BaseCommand +from geonode.geoserver.helpers import gs_catalog + +from geonode.base.models import ResourceBase + +logger = logging.getLogger() +handler = logging.StreamHandler(sys.stdout) +logger.setLevel(logging.INFO) +formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") +handler.setFormatter(formatter) +logger.addHandler(handler) + + +class Command(BaseCommand): + + help = """ + Migrate the files from MEDIA_ROOT to ASSETS_ROOT, update LocalAsset.location and GeoServer + REF: https://github.com/GeoNode/geonode/issues/12326 + """ + + def add_arguments(self, parser): + parser.add_argument( + '-n', + '--no-input', + dest='noinput', + default=False, + action='store_true', + help='Does not ask for confirmation for the run' + ) + parser.add_argument( + '-d', + '--dryrun', + dest='dryrun', + default=False, + action='store_true', + help='Perform a dryrun, will show the File to be moved, their new path and the file to be deleted' + ) + + def handle(self, **options): + question = "By running this command you are going to move all the files of your Resources into the ASSETS_ROOT. Do you want to continue?" + + if not options.get('noinput'): + result = confirm(question, resp=True) + if not result: + return + + dryrun = options.get('dryrun') + + handler = LocalAssetHandler() + + logger.info("Retrieving all assets with some files") + + for asset in LocalAsset.objects.iterator(): + logger.info(f"processing asset: {asset.title}") + + source = os.path.dirname(asset.location[0]) + + if dryrun: + logger.info(f"Files found: {asset.location}") + continue + + if settings.ASSETS_ROOT in source: + logger.info("The location is already the asset root, skipping...") + continue + + if not os.path.exists(source): + logger.warning("Source path of the file for Asset does not exists, skipping...") + continue + + try: + + logger.info("Moving file to the asset folder") + + dest = shutil.move(source, handler._create_asset_dir()) + + logger.info("Fixing perms") + if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None: + os.chmod(os.path.dirname(dest), settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS) + + if settings.FILE_UPLOAD_PERMISSIONS is not None: + os.chmod(dest, settings.FILE_UPLOAD_PERMISSIONS) + + except Exception as e: + logger.error(e) + continue + + logger.info("Updating location field with new folder value") + + asset.location = [x.replace(source, dest) for x in asset.location] + asset.save() + + logger.info("Checking if geoserver should be updated") + asset_obj = asset.link_set.values_list('resource', flat=True).first() + if not asset_obj: + logger.warning("No resources connected to the asset, skipping resource update...") + continue + resource = ResourceBase.objects.get(pk=asset_obj) + if resource.subtype == 'raster': + logger.info("Updating GeoServer value") + + store_to_update = gs_catalog.get_layer(resource.get_real_instance().alternate)\ + .resource\ + .store + + raster_file = [x for x in asset.location if os.path.basename(x).split('.')[-1] in ["tiff", "tif", "geotiff", "geotif"]] + store_to_update.url = f"file:{raster_file[0]}" + try: + gs_catalog.save(store_to_update) + except Exception: + logger.error(f"Error during GeoServer update for resource {resource}, please check GeoServer logs") + logger.info("Geoserver Updated") + + logger.info("Migration completed")