From 09ee8c229f83a0ca373d54a8a36a0d0c2a8265e2 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 27 Aug 2023 22:16:37 -0400 Subject: [PATCH 1/5] fix dataset delection (#445) --- wis2box-management/wis2box/api/__init__.py | 28 ++++++++++------- wis2box-management/wis2box/api/config/base.py | 22 +++++++++++++ .../wis2box/api/config/pygeoapi.py | 31 +++++++++++++++++++ wis2box-management/wis2box/data/__init__.py | 16 +++++----- 4 files changed, 76 insertions(+), 21 deletions(-) diff --git a/wis2box-management/wis2box/api/__init__.py b/wis2box-management/wis2box/api/__init__.py index d37d9948..ffc6aa3f 100644 --- a/wis2box-management/wis2box/api/__init__.py +++ b/wis2box-management/wis2box/api/__init__.py @@ -69,12 +69,12 @@ def setup_collection(meta: dict = {}) -> bool: return True -def remove_collection(name: str, backend: bool = True, +def remove_collection(collection_id: str, backend: bool = True, config: bool = True) -> bool: """ Remove collection from api backend and configuration - :param name: `str` of collection name + :param collection_id: `str` of collection id :returns: `bool` of API collection removal result """ @@ -82,24 +82,28 @@ def remove_collection(name: str, backend: bool = True, api_backend = None api_config = None - if backend: - api_backend = load_backend() - if api_backend.has_collection(name): - api_backend.delete_collection(name) + collection_data = load_config().get_collection_data(collection_id) if config: api_config = load_config() - if api_config.has_collection(name): - api_config.delete_collection(name) + if api_config.has_collection(collection_id): + api_config.delete_collection(collection_id) + + if backend: + api_backend = load_backend() + if api_backend.has_collection(collection_data): + api_backend.delete_collection(collection_data) - if api_backend is not None and api_backend.has_collection(name): - LOGGER.error(f'Unable to remove collection for {name}') + if api_backend is not None and api_backend.has_collection(collection_data): + LOGGER.error(f'Unable to remove collection backend for {collection_id}') return False - if api_config is not None and api_backend.has_collection(name): - LOGGER.error(f'Unable to remove collection for {name}') + if api_config is not None and api_config.has_collection(collection_data): + LOGGER.error(f'Unable to remove collection for {collection_id}') return False + delete_collection_item('discovery-metadata', collection_id) + return True diff --git a/wis2box-management/wis2box/api/config/base.py b/wis2box-management/wis2box/api/config/base.py index 22484961..9133634b 100644 --- a/wis2box-management/wis2box/api/config/base.py +++ b/wis2box-management/wis2box/api/config/base.py @@ -33,6 +33,28 @@ def __init__(self, defs: dict) -> None: :param defs: `dict` of connection parameters """ + def get_collection(self, name: str) -> dict: + """ + Get a collection + + :param name: `str` of collection name + + :returns: `dict` of collection configuration + """ + + raise NotImplementedError() + + def get_collection_data(self, name: str) -> dict: + """ + Get a collection's backend data configuration + + :param name: `str` of collection name + + :returns: `str` of collection backend data configuration + """ + + raise NotImplementedError() + def add_collection(self, name: str, collection: dict) -> bool: """ Add a collection diff --git a/wis2box-management/wis2box/api/config/pygeoapi.py b/wis2box-management/wis2box/api/config/pygeoapi.py index 646736d4..7c928327 100644 --- a/wis2box-management/wis2box/api/config/pygeoapi.py +++ b/wis2box-management/wis2box/api/config/pygeoapi.py @@ -20,6 +20,9 @@ ############################################################################### import logging +from urllib.parse import urlparse + + from requests import Session from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry @@ -52,6 +55,34 @@ def __init__(self, defs: dict) -> None: self.http.mount('https://', adapter) self.http.mount('http://', adapter) + def get_collection(self, name: str) -> dict: + """ + Get a collection + + :param name: `str` of collection name + + :returns: `dict` of collection configuration + """ + + r = self.http.get(f'{self.url}/{name}') + r.raise_for_status() + + return r.json() + + def get_collection_data(self, name: str) -> dict: + """ + Get a collection's backend data configuration + + :param name: `str` of collection name + + :returns: `str` of collection backend data configuration + """ + + data = self.get_collection(name)['providers'][0]['data'] + + collection_data = urlparse(data).path.lstrip('/') + return collection_data + def add_collection(self, name: str, collection: dict) -> bool: """ Add a collection diff --git a/wis2box-management/wis2box/data/__init__.py b/wis2box-management/wis2box/data/__init__.py index cc39f582..7f8bc073 100644 --- a/wis2box-management/wis2box/data/__init__.py +++ b/wis2box-management/wis2box/data/__init__.py @@ -35,7 +35,6 @@ from wis2box.metadata.discovery import DiscoveryMetadata from wis2box.storage import put_data, move_data, list_content, delete_data from wis2box.util import older_than, walk_path -from wis2box.topic_hierarchy import validate_and_load LOGGER = logging.getLogger(__name__) @@ -220,18 +219,17 @@ def add_collection(ctx, filepath, verbosity): @click.command() @click.pass_context -@cli_helpers.OPTION_TOPIC_HIERARCHY +@click.option('--identifier', '-i', help='Collection identifier') @cli_helpers.OPTION_VERBOSITY -def delete_collection(ctx, topic_hierarchy, verbosity): - """Delete collection from api backend""" +def delete_collection(ctx, identifier, verbosity): + """Delete collection from API backend""" - if topic_hierarchy is None: - raise click.ClickException('Missing -th/--topic-hierarchy') + if identifier is None: + raise click.ClickException('Missing -i/--identifier') - click.echo(f'Deleting collection: {topic_hierarchy}') + click.echo(f'Deleting collection: {identifier}') - th, _ = validate_and_load(topic_hierarchy) - remove_collection(th.dotpath) + remove_collection(identifier) click.echo('Done') From 0ae2b146553ef5dc5bd62867161ab5f885462114 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 27 Aug 2023 22:22:55 -0400 Subject: [PATCH 2/5] update dev docker setup --- docker-compose.dev.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3d219b63..f9177ef1 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,14 +1,14 @@ services: - wis2box: + wis2box-management: volumes: - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw - ./wis2box-management/wis2box/wis2box.cron:/etc/cron.d/wis2box:ro - - ./wis2box-management/wis2box:/usr/local/lib/python3.9/site-packages/wis2box-0.3.dev0-py3.9.egg/wis2box + - ./wis2box-management/wis2box:/app/wis2box command: ["wis2box", "pubsub" , "subscribe", "--broker", "http://wis2box-minio:9000", "--topic", "wis2box-storage/#", "--verbosity", "INFO"] -# wis2box-api: -# volumes: -# - ../../wis2box-api/wis2box_api:/usr/local/lib/python3.8/dist-packages/wis2box_api-0.3.dev0-py3.8.egg/wis2box_api + wis2box-api: + volumes: + - ../wis2box-api/wis2box_api:/app/wis2box_api elasticsearch: ports: From e5f60ba6acb9a706e1f3d18469bbca339ad9b94a Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 27 Aug 2023 22:36:56 -0400 Subject: [PATCH 3/5] update CLI and docs --- docs/source/reference/running/api-publishing.rst | 8 ++++++-- wis2box-management/wis2box/data/__init__.py | 11 ++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/source/reference/running/api-publishing.rst b/docs/source/reference/running/api-publishing.rst index 738b5069..e8ea68aa 100644 --- a/docs/source/reference/running/api-publishing.rst +++ b/docs/source/reference/running/api-publishing.rst @@ -51,11 +51,15 @@ discovery metadata MCF created as described in the :ref:`discovery-metadata` sec wis2box data add-collection $WIS2BOX_DATADIR/data/config/foo/bar/baz/discovery-metadata.yml -To delete the colection from the API backend and configuration: + +Deleting a dataset +------------------ + +To delete a dataset from the API backend and configuration: .. code-block:: bash - wis2box api delete-collection foo.bar.baz + wis2box data delete-collection dataset-id .. note:: diff --git a/wis2box-management/wis2box/data/__init__.py b/wis2box-management/wis2box/data/__init__.py index 7f8bc073..c1385956 100644 --- a/wis2box-management/wis2box/data/__init__.py +++ b/wis2box-management/wis2box/data/__init__.py @@ -219,17 +219,14 @@ def add_collection(ctx, filepath, verbosity): @click.command() @click.pass_context -@click.option('--identifier', '-i', help='Collection identifier') +@click.argument('collection') @cli_helpers.OPTION_VERBOSITY -def delete_collection(ctx, identifier, verbosity): +def delete_collection(ctx, collection, verbosity): """Delete collection from API backend""" - if identifier is None: - raise click.ClickException('Missing -i/--identifier') + click.echo(f'Deleting collection: {collection}') - click.echo(f'Deleting collection: {identifier}') - - remove_collection(identifier) + remove_collection(collection) click.echo('Done') From e26a5cc733522ce8f6a4b63a3453593a5beb3e86 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 28 Aug 2023 06:57:14 -0400 Subject: [PATCH 4/5] test sleep --- .github/workflows/tests-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 4dabcdd2..d0dc52df 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -40,7 +40,7 @@ jobs: docker logs wis2box-management - name: setup wis2box-management ⚙️ run: | - sleep 30 + sleep 60 python3 wis2box-ctl.py execute wis2box environment show - name: add Malawi data 🇲🇼 env: From e020e26aedfbded1ff79b5465050fec3ba311271 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 28 Aug 2023 12:18:46 -0400 Subject: [PATCH 5/5] fix WCMP2 deletion workflow --- .github/workflows/tests-docker.yml | 2 +- wis2box-management/wis2box/api/__init__.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index d0dc52df..4dabcdd2 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -40,7 +40,7 @@ jobs: docker logs wis2box-management - name: setup wis2box-management ⚙️ run: | - sleep 60 + sleep 30 python3 wis2box-ctl.py execute wis2box environment show - name: add Malawi data 🇲🇼 env: diff --git a/wis2box-management/wis2box/api/__init__.py b/wis2box-management/wis2box/api/__init__.py index ffc6aa3f..2d439cf1 100644 --- a/wis2box-management/wis2box/api/__init__.py +++ b/wis2box-management/wis2box/api/__init__.py @@ -95,14 +95,16 @@ def remove_collection(collection_id: str, backend: bool = True, api_backend.delete_collection(collection_data) if api_backend is not None and api_backend.has_collection(collection_data): - LOGGER.error(f'Unable to remove collection backend for {collection_id}') + msg = f'Unable to remove collection backend for {collection_id}' + LOGGER.error(msg) return False if api_config is not None and api_config.has_collection(collection_data): LOGGER.error(f'Unable to remove collection for {collection_id}') return False - delete_collection_item('discovery-metadata', collection_id) + if collection_id not in ['discovery-metadata', 'stations', 'messages']: + delete_collection_item('discovery-metadata', collection_id) return True