diff --git a/.travis/before_install.sh b/.travis/before_install.sh index 18618b4f01..7e5ef54802 100755 --- a/.travis/before_install.sh +++ b/.travis/before_install.sh @@ -101,7 +101,7 @@ if [ -z "$TRAVIS_TAG" ]; then fi -git clone --depth=1 https://github.com/pulp/pulp_ansible.git --branch 0.4.2 +git clone --depth=1 https://github.com/pulp/pulp_ansible.git --branch 0.5.0 if [ -n "$PULP_ANSIBLE_PR_NUMBER" ]; then cd pulp_ansible git fetch --depth=1 origin pull/$PULP_ANSIBLE_PR_NUMBER/head:$PULP_ANSIBLE_PR_NUMBER diff --git a/CHANGES/76.feature b/CHANGES/76.feature new file mode 100644 index 0000000000..1e8e056a42 --- /dev/null +++ b/CHANGES/76.feature @@ -0,0 +1 @@ +Support pulp_ansible collection deprecation edits diff --git a/CHANGES/81.misc b/CHANGES/81.misc new file mode 100644 index 0000000000..935aaf0420 --- /dev/null +++ b/CHANGES/81.misc @@ -0,0 +1 @@ +Update import tasks to maintain collection deprecation data diff --git a/CHANGES/82.misc b/CHANGES/82.misc new file mode 100644 index 0000000000..df1453b920 --- /dev/null +++ b/CHANGES/82.misc @@ -0,0 +1 @@ +Update repo move tasks to maintain collection deprecation data diff --git a/galaxy_ng/app/api/ui/serializers/collection.py b/galaxy_ng/app/api/ui/serializers/collection.py index 3970d033ba..fee028fa26 100644 --- a/galaxy_ng/app/api/ui/serializers/collection.py +++ b/galaxy_ng/app/api/ui/serializers/collection.py @@ -112,28 +112,15 @@ class _CollectionSerializer(Serializer): name = serializers.CharField() download_count = serializers.IntegerField(default=0) latest_version = serializers.SerializerMethodField() - deprecated = serializers.SerializerMethodField() def get_namespace(self, obj): namespace = Namespace.objects.get(name=obj.namespace) return NamespaceSummarySerializer(namespace).data - def get_deprecated(self, obj): - return obj.collection.deprecated - - def _get_versions_in_repo(self, obj): - distro = AnsibleDistribution.objects.get(base_path=self.context['path']) - repository_version = distro.repository.latest_version() - versions_in_repo = CollectionVersion.objects.filter( - pk__in=repository_version.content, - collection=obj.collection, - ) - return sorted( - versions_in_repo, key=lambda obj: semantic_version.Version(obj.version), reverse=True - ) - class CollectionListSerializer(_CollectionSerializer): + deprecated = serializers.BooleanField() + def get_latest_version(self, obj): return CollectionVersionBaseSerializer(obj).data @@ -141,11 +128,20 @@ def get_latest_version(self, obj): class CollectionDetailSerializer(_CollectionSerializer): all_versions = serializers.SerializerMethodField() - def get_all_versions(self, obj): - versions_in_repo = self._get_versions_in_repo(obj) - return CollectionVersionSummarySerializer(versions_in_repo, many=True).data - - # TODO(awcrosby): rename field to "version_details" since with + # TODO: rename field to "version_details" since with # "version" query param this won't always be the latest version def get_latest_version(self, obj): return CollectionVersionDetailSerializer(obj).data + + def get_all_versions(self, obj): + path = self.context['request'].parser_context['kwargs']['path'] + distro = AnsibleDistribution.objects.get(base_path=path) + repository_version = distro.repository.latest_version() + versions_in_repo = CollectionVersion.objects.filter( + pk__in=repository_version.content, + collection=obj.collection, + ) + versions_in_repo = sorted( + versions_in_repo, key=lambda obj: semantic_version.Version(obj.version), reverse=True + ) + return CollectionVersionSummarySerializer(versions_in_repo, many=True).data diff --git a/galaxy_ng/app/api/ui/viewsets/collection.py b/galaxy_ng/app/api/ui/viewsets/collection.py index 2fb015f230..f0671cc1b0 100644 --- a/galaxy_ng/app/api/ui/viewsets/collection.py +++ b/galaxy_ng/app/api/ui/viewsets/collection.py @@ -1,3 +1,4 @@ +from django.db.models import Exists, OuterRef, Q from django.core.exceptions import ObjectDoesNotExist from django.shortcuts import get_object_or_404 from django_filters import filters @@ -6,14 +7,17 @@ from pulp_ansible.app.galaxy.v3 import views as pulp_ansible_galaxy_views from pulp_ansible.app import viewsets as pulp_ansible_viewsets from pulp_ansible.app.models import ( + AnsibleCollectionDeprecated, AnsibleDistribution, CollectionVersion, Collection, CollectionRemote, ) from pulp_ansible.app.models import CollectionImport as PulpCollectionImport +from rest_framework import mixins from rest_framework.exceptions import NotFound from rest_framework.response import Response +import semantic_version from galaxy_ng.app.api import base as api_base from galaxy_ng.app.access_control import access_policy @@ -21,23 +25,59 @@ from galaxy_ng.app.api.v3.serializers.sync import CollectionRemoteSerializer -class CollectionFilter(pulp_ansible_viewsets.CollectionVersionFilter): +class CollectionByCollectionVersionFilter(pulp_ansible_viewsets.CollectionVersionFilter): """pulp_ansible CollectionVersion filter for Collection viewset.""" versioning_class = versioning.UIVersioning keywords = filters.CharFilter(field_name="keywords", method="filter_by_q") + deprecated = filters.BooleanFilter() -class CollectionViewSet(api_base.LocalSettingsMixin, pulp_ansible_galaxy_views.CollectionViewSet): +class CollectionViewSet( + api_base.GenericViewSet, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + pulp_ansible_galaxy_views.AnsibleDistributionMixin, +): """Viewset that uses CollectionVersion's within distribution to display data for Collection's. - Collection list is filterable by CollectionFilter and includes latest CollectionVersion. + Collection list is filterable by FilterSet and includes latest CollectionVersion. Collection detail includes CollectionVersion that is latest or via query param 'version'. """ versioning_class = versioning.UIVersioning - filterset_class = CollectionFilter + filterset_class = CollectionByCollectionVersionFilter permission_classes = [access_policy.CollectionAccessPolicy] + def get_queryset(self): + """Returns a CollectionVersions queryset for specified distribution.""" + distro_content = self.get_distro_content(self.kwargs["path"]) + repo_version = self.get_repository_version(self.kwargs["path"]) + + versions = CollectionVersion.objects.filter(pk__in=distro_content).values_list( + "collection_id", + "version", + ) + + collection_versions = {} + for collection_id, version in versions: + value = collection_versions.get(str(collection_id)) + if not value or semantic_version.Version(version) > semantic_version.Version(value): + collection_versions[str(collection_id)] = version + + if not collection_versions.items(): + return CollectionVersion.objects.none() + + query_params = Q() + for collection_id, version in collection_versions.items(): + query_params |= Q(collection_id=collection_id, version=version) + + deprecated_query = AnsibleCollectionDeprecated.objects.filter( + collection=OuterRef("collection"), repository_version=repo_version + ) + version_qs = CollectionVersion.objects.select_related("collection").filter(query_params) + version_qs = version_qs.annotate(deprecated=Exists(deprecated_query)) + return version_qs + def get_object(self): """Return CollectionVersion object, latest or via query param 'version'.""" version = self.request.query_params.get('version', None) diff --git a/galaxy_ng/app/api/v3/serializers/collection.py b/galaxy_ng/app/api/v3/serializers/collection.py index 93dfd6af1c..bcee265e63 100644 --- a/galaxy_ng/app/api/v3/serializers/collection.py +++ b/galaxy_ng/app/api/v3/serializers/collection.py @@ -53,14 +53,15 @@ def get_versions_url(self, obj): def get_highest_version(self, obj): """Get a highest version and its link.""" + collection = self.context["highest_versions"][obj.pk] kwargs = { "path": self.context["path"], - "namespace": obj.namespace, - "name": obj.name, - "version": obj.version, + "namespace": collection.namespace, + "name": collection.name, + "version": collection.version, } href = self._get_href("collection-versions-detail", **kwargs) - return {"href": href, "version": obj.version} + return {"href": href, "version": str(collection.version)} class CollectionRefSerializer(_CollectionRefSerializer, HrefNamespaceMixin): diff --git a/galaxy_ng/app/api/v3/urls.py b/galaxy_ng/app/api/v3/urls.py index 9455381d09..10900e9d23 100644 --- a/galaxy_ng/app/api/v3/urls.py +++ b/galaxy_ng/app/api/v3/urls.py @@ -38,7 +38,7 @@ ), path( 'collections///', - viewsets.CollectionViewSet.as_view({'get': 'retrieve', 'put': 'update'}), + viewsets.CollectionViewSet.as_view({'get': 'retrieve', 'patch': 'update'}), name='collections-detail' ), path( diff --git a/galaxy_ng/app/api/v3/viewsets/collection.py b/galaxy_ng/app/api/v3/viewsets/collection.py index 4b63d709bd..61c00f3e51 100644 --- a/galaxy_ng/app/api/v3/viewsets/collection.py +++ b/galaxy_ng/app/api/v3/viewsets/collection.py @@ -39,8 +39,8 @@ from galaxy_ng.app.tasks import ( import_and_move_to_staging, import_and_auto_approve, - add_content_to_repository, - remove_content_from_repository, + call_copy_task, + call_remove_task, curate_all_synclist_repository, ) @@ -316,8 +316,8 @@ def move_content(self, request, *args, **kwargs): if collection_version in dest_versions: raise NotFound(f'Collection {version_str} already found in destination repo') - add_task = self._add_content(collection_version, dest_repo) - remove_task = self._remove_content(collection_version, src_repo) + copy_task = call_copy_task(collection_version, src_repo, dest_repo) + remove_task = call_remove_task(collection_version, src_repo) curate_task_id = None if settings.GALAXY_DEPLOYMENT_MODE == DeploymentMode.INSIGHTS.value: @@ -338,21 +338,9 @@ def move_content(self, request, *args, **kwargs): return Response( data={ - 'add_task_id': add_task.id, + 'copy_task_id': copy_task.id, 'remove_task_id': remove_task.id, "curate_all_synclist_repository_task_id": curate_task_id, }, status='202' ) - - @staticmethod - def _add_content(collection_version, repo): - locks = [repo] - task_args = (collection_version.pk, repo.pk) - return enqueue_with_reservation(add_content_to_repository, locks, args=task_args) - - @staticmethod - def _remove_content(collection_version, repo): - locks = [repo] - task_args = (collection_version.pk, repo.pk) - return enqueue_with_reservation(remove_content_from_repository, locks, args=task_args) diff --git a/galaxy_ng/app/tasks/__init__.py b/galaxy_ng/app/tasks/__init__.py index 4c9aa373dd..3d52dabb28 100644 --- a/galaxy_ng/app/tasks/__init__.py +++ b/galaxy_ng/app/tasks/__init__.py @@ -1,4 +1,4 @@ from .publishing import import_and_move_to_staging, import_and_auto_approve # noqa: F401 -from .promotion import add_content_to_repository, remove_content_from_repository # noqa: F401 +from .promotion import call_copy_task, call_remove_task # noqa: F401 from .synclist import curate_synclist_repository, curate_all_synclist_repository # noqa: F401 # from .synchronizing import synchronize # noqa diff --git a/galaxy_ng/app/tasks/promotion.py b/galaxy_ng/app/tasks/promotion.py index 2be93992ca..23983f7358 100644 --- a/galaxy_ng/app/tasks/promotion.py +++ b/galaxy_ng/app/tasks/promotion.py @@ -1,20 +1,36 @@ +from pulpcore.plugin.tasking import enqueue_with_reservation from pulp_ansible.app.models import AnsibleRepository, CollectionVersion +from pulp_ansible.app.tasks.copy import copy_content -def add_content_to_repository(collection_version_pk, repository_pk): - """ - Add a CollectionVersion to repository. - Args: - collection_version_pk: The pk of the CollectionVersion to add to repository. - repository_pk: The pk of the AnsibleRepository to add the CollectionVersion to. - """ - repository = AnsibleRepository.objects.get(pk=repository_pk) - qs = CollectionVersion.objects.filter(pk=collection_version_pk) - with repository.new_version() as new_version: - new_version.add_content(qs) +def call_copy_task(collection_version, source_repo, dest_repo): + """Calls pulp_ansible task to copy content from source to destination repo.""" + locks = [source_repo, dest_repo] + config = [{ + 'source_repo_version': source_repo.latest_version().pk, + 'dest_repo': dest_repo.pk, + 'content': [collection_version.pk], + }] + return enqueue_with_reservation( + copy_content, + locks, + args=[config], + kwargs={}, + ) + + +def call_remove_task(collection_version, repository): + """Calls task to remove content from repo.""" + remove_task_args = (collection_version.pk, repository.pk) + return enqueue_with_reservation( + _remove_content_from_repository, + [repository], + args=remove_task_args, + kwargs={}, + ) -def remove_content_from_repository(collection_version_pk, repository_pk): +def _remove_content_from_repository(collection_version_pk, repository_pk): """ Remove a CollectionVersion from a repository. Args: diff --git a/galaxy_ng/app/tasks/publishing.py b/galaxy_ng/app/tasks/publishing.py index 0ae06ca34d..8187fbb849 100644 --- a/galaxy_ng/app/tasks/publishing.py +++ b/galaxy_ng/app/tasks/publishing.py @@ -4,11 +4,10 @@ from django.contrib.contenttypes.models import ContentType from pulpcore.plugin.models import Task -from pulpcore.plugin.tasking import enqueue_with_reservation from pulp_ansible.app.models import AnsibleDistribution, AnsibleRepository, CollectionVersion from pulp_ansible.app.tasks.collections import import_collection -from .promotion import add_content_to_repository, remove_content_from_repository +from .promotion import call_copy_task, call_remove_task log = logging.getLogger(__name__) @@ -50,20 +49,11 @@ def import_and_move_to_staging(temp_file_pk, **kwargs): inbound_repo = AnsibleRepository.objects.get(pk=inbound_repository_pk) - inbound_locks = [inbound_repo] - staging_locks = [staging_repo] - created_collection_versions = get_created_collection_versions() for collection_version in created_collection_versions: - # enqueue task to add collection_version to staging repo - add_task_args = (collection_version.pk, staging_repo.pk) - enqueue_with_reservation(add_content_to_repository, staging_locks, args=add_task_args) - - # enqueue task to remove collection_verion from inbound repo - remove_task_args = (collection_version.pk, inbound_repository_pk) - enqueue_with_reservation(remove_content_from_repository, - inbound_locks, args=remove_task_args) + call_copy_task(collection_version, inbound_repo, staging_repo) + call_remove_task(collection_version, inbound_repo) def import_and_auto_approve(temp_file_pk, **kwargs): @@ -83,20 +73,11 @@ def import_and_auto_approve(temp_file_pk, **kwargs): inbound_repo = AnsibleRepository.objects.get(pk=inbound_repository_pk) - remove_locks = [inbound_repo] - add_locks = [golden_repo] - created_collection_versions = get_created_collection_versions() for collection_version in created_collection_versions: - # enqueue task to add collection_version to golden repo - add_task_args = (collection_version.pk, golden_repo.pk) - enqueue_with_reservation(add_content_to_repository, add_locks, args=add_task_args) - - # enqueue task to remove collection_verion from inbound repo - remove_task_args = (collection_version.pk, inbound_repository_pk) - enqueue_with_reservation(remove_content_from_repository, - remove_locks, args=remove_task_args) + call_copy_task(collection_version, inbound_repo, golden_repo) + call_remove_task(collection_version, inbound_repo) log.info('Imported and auto approved collection artifact %s to repository %s', collection_version.relative_path, diff --git a/galaxy_ng/tests/unit/api/test_api_ui_sync_config.py b/galaxy_ng/tests/unit/api/test_api_ui_sync_config.py index 3c5a2ca2c0..ac635203b0 100644 --- a/galaxy_ng/tests/unit/api/test_api_ui_sync_config.py +++ b/galaxy_ng/tests/unit/api/test_api_ui_sync_config.py @@ -92,7 +92,7 @@ def test_positive_update_certified_repo_data(self): "name": "rh-certified", "policy": "immediate", "requirements_file": None, - "url": "https://updated.url.com", + "url": "https://updated.url.com/", }, format='json' ) @@ -103,7 +103,7 @@ def test_positive_update_certified_repo_data(self): updated = self.client.get(self.build_config_url(self.certified_remote.name)) self.assertEqual(updated.data["auth_url"], "https://auth.com") - self.assertEqual(updated.data["url"], "https://updated.url.com") + self.assertEqual(updated.data["url"], "https://updated.url.com/") self.assertIsNone(updated.data["requirements_file"]) def test_negative_update_community_repo_data_without_requirements_file(self): @@ -143,7 +143,7 @@ def test_positive_update_community_repo_data_with_requirements_file(self): " server: https://foobar.content.com\n" " api_key: s3cr3tk3y\n" ), - "url": "https://galaxy.ansible.com/v3/collections", + "url": "https://galaxy.ansible.com/v3/collections/", }, format='json' ) diff --git a/galaxy_ng/tests/unit/app/test_tasks.py b/galaxy_ng/tests/unit/app/test_tasks.py index 02a037a333..61cc4070fa 100644 --- a/galaxy_ng/tests/unit/app/test_tasks.py +++ b/galaxy_ng/tests/unit/app/test_tasks.py @@ -10,13 +10,10 @@ from pulp_ansible.app.models import ( Collection, CollectionVersion, AnsibleRepository, AnsibleDistribution ) +from pulp_ansible.app.tasks.copy import copy_content -from galaxy_ng.app.tasks import ( - add_content_to_repository, - remove_content_from_repository, - import_and_auto_approve, - import_and_move_to_staging, -) +from galaxy_ng.app.tasks import import_and_auto_approve, import_and_move_to_staging +from galaxy_ng.app.tasks.promotion import _remove_content_from_repository log = logging.getLogger(__name__) @@ -49,40 +46,63 @@ def setUp(self): ) content_artifact.save() - def test_add_content_to_repository(self): - repo = AnsibleRepository.objects.get(name=staging_name) - repo_version_number = repo.latest_version().number + def test_task_copy_content(self): + repo1 = AnsibleRepository.objects.get(name=staging_name) + repo1_version_number = repo1.latest_version().number + repo2 = AnsibleRepository.objects.get(name='rejected') + repo2_version_number = repo2.latest_version().number self.assertNotIn( self.collection_version, - CollectionVersion.objects.filter(pk__in=repo.latest_version().content)) + CollectionVersion.objects.filter(pk__in=repo1.latest_version().content)) + self.assertNotIn( + self.collection_version, + CollectionVersion.objects.filter(pk__in=repo2.latest_version().content)) - add_content_to_repository(self.collection_version.pk, repo.pk) + qs = CollectionVersion.objects.filter(pk=self.collection_version.pk) + with repo1.new_version() as new_version: + new_version.add_content(qs) - self.assertEqual(repo_version_number + 1, repo.latest_version().number) + self.assertEqual(repo1_version_number + 1, repo1.latest_version().number) self.assertIn( self.collection_version, - CollectionVersion.objects.filter(pk__in=repo.latest_version().content)) + CollectionVersion.objects.filter(pk__in=repo1.latest_version().content)) - def test_remove_content_from_repository(self): - repo = AnsibleRepository.objects.get(name=staging_name) - add_content_to_repository(self.collection_version.pk, repo.pk) + config = [{ + 'source_repo_version': repo1.latest_version().pk, + 'dest_repo': repo2.pk, + 'content': [self.collection_version.pk], + }] + copy_content(config) + self.assertEqual(repo2_version_number + 1, repo2.latest_version().number) + self.assertIn( + self.collection_version, + CollectionVersion.objects.filter(pk__in=repo2.latest_version().content)) + + def test_task_remove_content_from_repository(self): + repo = AnsibleRepository.objects.get(name=staging_name) repo_version_number = repo.latest_version().number + + qs = CollectionVersion.objects.filter(pk=self.collection_version.pk) + with repo.new_version() as new_version: + new_version.add_content(qs) + + self.assertEqual(repo_version_number + 1, repo.latest_version().number) self.assertIn( self.collection_version, CollectionVersion.objects.filter(pk__in=repo.latest_version().content)) - remove_content_from_repository(self.collection_version.pk, repo.pk) + _remove_content_from_repository(self.collection_version.pk, repo.pk) - self.assertEqual(repo_version_number + 1, repo.latest_version().number) + self.assertEqual(repo_version_number + 2, repo.latest_version().number) self.assertNotIn( self.collection_version, CollectionVersion.objects.filter(pk__in=repo.latest_version().content)) @mock.patch('galaxy_ng.app.tasks.publishing.get_created_collection_versions') @mock.patch('galaxy_ng.app.tasks.publishing.import_collection') - @mock.patch('galaxy_ng.app.tasks.publishing.enqueue_with_reservation') + @mock.patch('galaxy_ng.app.tasks.promotion.enqueue_with_reservation') def test_import_and_auto_approve(self, mocked_enqueue, mocked_import, mocked_get_created): inbound_repo = AnsibleRepository.objects.get(name=staging_name) @@ -107,7 +127,7 @@ def test_import_and_auto_approve(self, mocked_enqueue, mocked_import, mocked_get @mock.patch('galaxy_ng.app.tasks.publishing.get_created_collection_versions') @mock.patch('galaxy_ng.app.tasks.publishing.import_collection') - @mock.patch('galaxy_ng.app.tasks.publishing.enqueue_with_reservation') + @mock.patch('galaxy_ng.app.tasks.promotion.enqueue_with_reservation') def test_import_and_move_to_staging(self, mocked_enqueue, mocked_import, mocked_get_created): staging_repo = AnsibleRepository.objects.get(name=staging_name) diff --git a/requirements/requirements.common.txt b/requirements/requirements.common.txt index 4809474ee9..2a4d6c8081 100644 --- a/requirements/requirements.common.txt +++ b/requirements/requirements.common.txt @@ -60,7 +60,7 @@ openpyxl==3.0.5 # via tablib packaging==20.4 # via ansible-base, bleach, pulp-ansible prometheus-client==0.8.0 # via django-prometheus psycopg2==2.8.6 # via pulpcore -pulp-ansible==0.4.2 # via galaxy-ng (setup.py) +pulp-ansible==0.5.0 # via galaxy-ng (setup.py) pulpcore==3.7.1 # via galaxy-ng (setup.py), pulp-ansible pycares==3.1.1 # via aiodns pycodestyle==2.6.0 # via flake8 diff --git a/requirements/requirements.insights.txt b/requirements/requirements.insights.txt index 2f134fe0de..bd6e953b98 100644 --- a/requirements/requirements.insights.txt +++ b/requirements/requirements.insights.txt @@ -65,7 +65,7 @@ openpyxl==3.0.5 # via tablib packaging==20.4 # via ansible-base, bleach, pulp-ansible prometheus-client==0.8.0 # via django-prometheus psycopg2==2.8.6 # via pulpcore -pulp-ansible==0.4.2 # via galaxy-ng (setup.py) +pulp-ansible==0.5.0 # via galaxy-ng (setup.py) pulpcore==3.7.1 # via galaxy-ng (setup.py), pulp-ansible pycares==3.1.1 # via aiodns pycodestyle==2.6.0 # via flake8 diff --git a/requirements/requirements.standalone.txt b/requirements/requirements.standalone.txt index 45e59c11fc..ef6bca8a38 100644 --- a/requirements/requirements.standalone.txt +++ b/requirements/requirements.standalone.txt @@ -62,7 +62,7 @@ openpyxl==3.0.5 # via tablib packaging==20.4 # via ansible-base, bleach, pulp-ansible prometheus-client==0.8.0 # via django-prometheus psycopg2==2.8.6 # via pulpcore -pulp-ansible==0.4.2 # via galaxy-ng (setup.py) +pulp-ansible==0.5.0 # via galaxy-ng (setup.py) pulp-container==2.1.0 # via -r requirements/requirements.standalone.in pulpcore==3.7.1 # via galaxy-ng (setup.py), pulp-ansible, pulp-container pycares==3.1.1 # via aiodns diff --git a/setup.py b/setup.py index 6113d0775e..d1b1c47390 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ def run(self): requirements = [ "Django~=2.2.3", "pulpcore>=3.7,<3.9", - "pulp-ansible~=0.4.2", + "pulp-ansible~=0.5.0", "django-prometheus>=2.0.0", "drf-spectacular", ]