Support pulp_ansible collection deprecation edits (#542)

* Fix v3 collection endpoint context for highest version
* Bring over CollectionViewSet using CollectionVersion qs from pulp_ansible
* Updates to allow filtering by deprecated
* Update url verb from put to patch for deprecation update
* Add changelog
* Use new copy task for promotion on certification and import
* Update tests for tasking
* Refactor serializer methods
* Update pulp_ansible to 0.5.0
* Fix tests stating 400 invalid urls needing to end in slash

Issue: AAH-76
Issue: AAH-81
Issue: AAH-82

.travis/before_install.sh

@@ -101,7 +101,7 @@ if [ -z "$TRAVIS_TAG" ]; then
 fi
 
 
-git clone --depth=1 https://github.com/pulp/pulp_ansible.git --branch 0.4.2
+git clone --depth=1 https://github.com/pulp/pulp_ansible.git --branch 0.5.0
 if [ -n "$PULP_ANSIBLE_PR_NUMBER" ]; then
   cd pulp_ansible
   git fetch --depth=1 origin pull/$PULP_ANSIBLE_PR_NUMBER/head:$PULP_ANSIBLE_PR_NUMBER

CHANGES/76.feature

@@ -0,0 +1 @@
+Support pulp_ansible collection deprecation edits

CHANGES/81.misc

@@ -0,0 +1 @@
+Update import tasks to maintain collection deprecation data

CHANGES/82.misc

@@ -0,0 +1 @@
+Update repo move tasks to maintain collection deprecation data

galaxy_ng/app/api/ui/serializers/collection.py

@@ -112,40 +112,36 @@ class _CollectionSerializer(Serializer):
     name = serializers.CharField()
     download_count = serializers.IntegerField(default=0)
     latest_version = serializers.SerializerMethodField()
-    deprecated = serializers.SerializerMethodField()
 
     def get_namespace(self, obj):
         namespace = Namespace.objects.get(name=obj.namespace)
         return NamespaceSummarySerializer(namespace).data
 
-    def get_deprecated(self, obj):
-        return obj.collection.deprecated
-
-    def _get_versions_in_repo(self, obj):
-        distro = AnsibleDistribution.objects.get(base_path=self.context['path'])
-        repository_version = distro.repository.latest_version()
-        versions_in_repo = CollectionVersion.objects.filter(
-            pk__in=repository_version.content,
-            collection=obj.collection,
-        )
-        return sorted(
-            versions_in_repo, key=lambda obj: semantic_version.Version(obj.version), reverse=True
-        )
-
 
 class CollectionListSerializer(_CollectionSerializer):
+    deprecated = serializers.BooleanField()
+
     def get_latest_version(self, obj):
         return CollectionVersionBaseSerializer(obj).data
 
 
 class CollectionDetailSerializer(_CollectionSerializer):
     all_versions = serializers.SerializerMethodField()
 
-    def get_all_versions(self, obj):
-        versions_in_repo = self._get_versions_in_repo(obj)
-        return CollectionVersionSummarySerializer(versions_in_repo, many=True).data
-
-    # TODO(awcrosby): rename field to "version_details" since with
+    # TODO: rename field to "version_details" since with
     # "version" query param this won't always be the latest version
     def get_latest_version(self, obj):
         return CollectionVersionDetailSerializer(obj).data
+
+    def get_all_versions(self, obj):
+        path = self.context['request'].parser_context['kwargs']['path']
+        distro = AnsibleDistribution.objects.get(base_path=path)
+        repository_version = distro.repository.latest_version()
+        versions_in_repo = CollectionVersion.objects.filter(
+            pk__in=repository_version.content,
+            collection=obj.collection,
+        )
+        versions_in_repo = sorted(
+            versions_in_repo, key=lambda obj: semantic_version.Version(obj.version), reverse=True
+        )
+        return CollectionVersionSummarySerializer(versions_in_repo, many=True).data

galaxy_ng/app/api/ui/viewsets/collection.py

@@ -1,3 +1,4 @@
+from django.db.models import Exists, OuterRef, Q
 from django.core.exceptions import ObjectDoesNotExist
 from django.shortcuts import get_object_or_404
 from django_filters import filters
@@ -6,38 +7,77 @@
 from pulp_ansible.app.galaxy.v3 import views as pulp_ansible_galaxy_views
 from pulp_ansible.app import viewsets as pulp_ansible_viewsets
 from pulp_ansible.app.models import (
+    AnsibleCollectionDeprecated,
     AnsibleDistribution,
     CollectionVersion,
     Collection,
     CollectionRemote,
 )
 from pulp_ansible.app.models import CollectionImport as PulpCollectionImport
+from rest_framework import mixins
 from rest_framework.exceptions import NotFound
 from rest_framework.response import Response
+import semantic_version
 
 from galaxy_ng.app.api import base as api_base
 from galaxy_ng.app.access_control import access_policy
 from galaxy_ng.app.api.ui import serializers, versioning
 from galaxy_ng.app.api.v3.serializers.sync import CollectionRemoteSerializer
 
 
-class CollectionFilter(pulp_ansible_viewsets.CollectionVersionFilter):
+class CollectionByCollectionVersionFilter(pulp_ansible_viewsets.CollectionVersionFilter):
     """pulp_ansible CollectionVersion filter for Collection viewset."""
     versioning_class = versioning.UIVersioning
     keywords = filters.CharFilter(field_name="keywords", method="filter_by_q")
+    deprecated = filters.BooleanFilter()
 
 
-class CollectionViewSet(api_base.LocalSettingsMixin, pulp_ansible_galaxy_views.CollectionViewSet):
+class CollectionViewSet(
+    api_base.GenericViewSet,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    pulp_ansible_galaxy_views.AnsibleDistributionMixin,
+):
     """Viewset that uses CollectionVersion's within distribution to display data for Collection's.
 
-    Collection list is filterable by CollectionFilter and includes latest CollectionVersion.
+    Collection list is filterable by FilterSet and includes latest CollectionVersion.
 
     Collection detail includes CollectionVersion that is latest or via query param 'version'.
     """
     versioning_class = versioning.UIVersioning
-    filterset_class = CollectionFilter
+    filterset_class = CollectionByCollectionVersionFilter
     permission_classes = [access_policy.CollectionAccessPolicy]
 
+    def get_queryset(self):
+        """Returns a CollectionVersions queryset for specified distribution."""
+        distro_content = self.get_distro_content(self.kwargs["path"])
+        repo_version = self.get_repository_version(self.kwargs["path"])
+
+        versions = CollectionVersion.objects.filter(pk__in=distro_content).values_list(
+            "collection_id",
+            "version",
+        )
+
+        collection_versions = {}
+        for collection_id, version in versions:
+            value = collection_versions.get(str(collection_id))
+            if not value or semantic_version.Version(version) > semantic_version.Version(value):
+                collection_versions[str(collection_id)] = version
+
+        if not collection_versions.items():
+            return CollectionVersion.objects.none()
+
+        query_params = Q()
+        for collection_id, version in collection_versions.items():
+            query_params |= Q(collection_id=collection_id, version=version)
+
+        deprecated_query = AnsibleCollectionDeprecated.objects.filter(
+            collection=OuterRef("collection"), repository_version=repo_version
+        )
+        version_qs = CollectionVersion.objects.select_related("collection").filter(query_params)
+        version_qs = version_qs.annotate(deprecated=Exists(deprecated_query))
+        return version_qs
+
     def get_object(self):
         """Return CollectionVersion object, latest or via query param 'version'."""
         version = self.request.query_params.get('version', None)

galaxy_ng/app/api/v3/serializers/collection.py

@@ -53,14 +53,15 @@ def get_versions_url(self, obj):
 
     def get_highest_version(self, obj):
         """Get a highest version and its link."""
+        collection = self.context["highest_versions"][obj.pk]
         kwargs = {
             "path": self.context["path"],
-            "namespace": obj.namespace,
-            "name": obj.name,
-            "version": obj.version,
+            "namespace": collection.namespace,
+            "name": collection.name,
+            "version": collection.version,
         }
         href = self._get_href("collection-versions-detail", **kwargs)
-        return {"href": href, "version": obj.version}
+        return {"href": href, "version": str(collection.version)}
 
 
 class CollectionRefSerializer(_CollectionRefSerializer, HrefNamespaceMixin):

galaxy_ng/app/api/v3/urls.py

@@ -38,7 +38,7 @@
     ),
     path(
         'collections/<str:namespace>/<str:name>/',
-        viewsets.CollectionViewSet.as_view({'get': 'retrieve', 'put': 'update'}),
+        viewsets.CollectionViewSet.as_view({'get': 'retrieve', 'patch': 'update'}),
         name='collections-detail'
     ),
     path(

galaxy_ng/app/api/v3/viewsets/collection.py

@@ -39,8 +39,8 @@
 from galaxy_ng.app.tasks import (
     import_and_move_to_staging,
     import_and_auto_approve,
-    add_content_to_repository,
-    remove_content_from_repository,
+    call_copy_task,
+    call_remove_task,
     curate_all_synclist_repository,
 )
 
@@ -316,8 +316,8 @@ def move_content(self, request, *args, **kwargs):
         if collection_version in dest_versions:
             raise NotFound(f'Collection {version_str} already found in destination repo')
 
-        add_task = self._add_content(collection_version, dest_repo)
-        remove_task = self._remove_content(collection_version, src_repo)
+        copy_task = call_copy_task(collection_version, src_repo, dest_repo)
+        remove_task = call_remove_task(collection_version, src_repo)
 
         curate_task_id = None
         if settings.GALAXY_DEPLOYMENT_MODE == DeploymentMode.INSIGHTS.value:
@@ -338,21 +338,9 @@ def move_content(self, request, *args, **kwargs):
 
         return Response(
             data={
-                'add_task_id': add_task.id,
+                'copy_task_id': copy_task.id,
                 'remove_task_id': remove_task.id,
                 "curate_all_synclist_repository_task_id": curate_task_id,
             },
             status='202'
         )
-
-    @staticmethod
-    def _add_content(collection_version, repo):
-        locks = [repo]
-        task_args = (collection_version.pk, repo.pk)
-        return enqueue_with_reservation(add_content_to_repository, locks, args=task_args)
-
-    @staticmethod
-    def _remove_content(collection_version, repo):
-        locks = [repo]
-        task_args = (collection_version.pk, repo.pk)
-        return enqueue_with_reservation(remove_content_from_repository, locks, args=task_args)

galaxy_ng/app/tasks/__init__.py

@@ -1,4 +1,4 @@
 from .publishing import import_and_move_to_staging, import_and_auto_approve  # noqa: F401
-from .promotion import add_content_to_repository, remove_content_from_repository  # noqa: F401
+from .promotion import call_copy_task, call_remove_task  # noqa: F401
 from .synclist import curate_synclist_repository, curate_all_synclist_repository  # noqa: F401
 # from .synchronizing import synchronize  # noqa

galaxy_ng/app/tasks/promotion.py

@@ -1,20 +1,36 @@
+from pulpcore.plugin.tasking import enqueue_with_reservation
 from pulp_ansible.app.models import AnsibleRepository, CollectionVersion
+from pulp_ansible.app.tasks.copy import copy_content
 
 
-def add_content_to_repository(collection_version_pk, repository_pk):
-    """
-    Add a CollectionVersion to repository.
-    Args:
-        collection_version_pk: The pk of the CollectionVersion to add to repository.
-        repository_pk: The pk of the AnsibleRepository to add the CollectionVersion to.
-    """
-    repository = AnsibleRepository.objects.get(pk=repository_pk)
-    qs = CollectionVersion.objects.filter(pk=collection_version_pk)
-    with repository.new_version() as new_version:
-        new_version.add_content(qs)
+def call_copy_task(collection_version, source_repo, dest_repo):
+    """Calls pulp_ansible task to copy content from source to destination repo."""
+    locks = [source_repo, dest_repo]
+    config = [{
+        'source_repo_version': source_repo.latest_version().pk,
+        'dest_repo': dest_repo.pk,
+        'content': [collection_version.pk],
+    }]
+    return enqueue_with_reservation(
+        copy_content,
+        locks,
+        args=[config],
+        kwargs={},
+    )
+
+
+def call_remove_task(collection_version, repository):
+    """Calls task to remove content from repo."""
+    remove_task_args = (collection_version.pk, repository.pk)
+    return enqueue_with_reservation(
+        _remove_content_from_repository,
+        [repository],
+        args=remove_task_args,
+        kwargs={},
+    )
 
 
-def remove_content_from_repository(collection_version_pk, repository_pk):
+def _remove_content_from_repository(collection_version_pk, repository_pk):
     """
     Remove a CollectionVersion from a repository.
     Args:

galaxy_ng/app/tasks/publishing.py

@@ -4,11 +4,10 @@
 from django.contrib.contenttypes.models import ContentType
 
 from pulpcore.plugin.models import Task
-from pulpcore.plugin.tasking import enqueue_with_reservation
 from pulp_ansible.app.models import AnsibleDistribution, AnsibleRepository, CollectionVersion
 from pulp_ansible.app.tasks.collections import import_collection
 
-from .promotion import add_content_to_repository, remove_content_from_repository
+from .promotion import call_copy_task, call_remove_task
 
 log = logging.getLogger(__name__)
 
@@ -50,20 +49,11 @@ def import_and_move_to_staging(temp_file_pk, **kwargs):
 
     inbound_repo = AnsibleRepository.objects.get(pk=inbound_repository_pk)
 
-    inbound_locks = [inbound_repo]
-    staging_locks = [staging_repo]
-
     created_collection_versions = get_created_collection_versions()
 
     for collection_version in created_collection_versions:
-        # enqueue task to add collection_version to staging repo
-        add_task_args = (collection_version.pk, staging_repo.pk)
-        enqueue_with_reservation(add_content_to_repository, staging_locks, args=add_task_args)
-
-        # enqueue task to remove collection_verion from inbound repo
-        remove_task_args = (collection_version.pk, inbound_repository_pk)
-        enqueue_with_reservation(remove_content_from_repository,
-                                 inbound_locks, args=remove_task_args)
+        call_copy_task(collection_version, inbound_repo, staging_repo)
+        call_remove_task(collection_version, inbound_repo)
 
 
 def import_and_auto_approve(temp_file_pk, **kwargs):
@@ -83,20 +73,11 @@ def import_and_auto_approve(temp_file_pk, **kwargs):
 
     inbound_repo = AnsibleRepository.objects.get(pk=inbound_repository_pk)
 
-    remove_locks = [inbound_repo]
-    add_locks = [golden_repo]
-
     created_collection_versions = get_created_collection_versions()
 
     for collection_version in created_collection_versions:
-        # enqueue task to add collection_version to golden repo
-        add_task_args = (collection_version.pk, golden_repo.pk)
-        enqueue_with_reservation(add_content_to_repository, add_locks, args=add_task_args)
-
-        # enqueue task to remove collection_verion from inbound repo
-        remove_task_args = (collection_version.pk, inbound_repository_pk)
-        enqueue_with_reservation(remove_content_from_repository,
-                                 remove_locks, args=remove_task_args)
+        call_copy_task(collection_version, inbound_repo, golden_repo)
+        call_remove_task(collection_version, inbound_repo)
 
         log.info('Imported and auto approved collection artifact %s to repository %s',
                  collection_version.relative_path,

galaxy_ng/tests/unit/api/test_api_ui_sync_config.py

@@ -92,7 +92,7 @@ def test_positive_update_certified_repo_data(self):
                 "name": "rh-certified",
                 "policy": "immediate",
                 "requirements_file": None,
-                "url": "https://updated.url.com",
+                "url": "https://updated.url.com/",
             },
             format='json'
         )
@@ -103,7 +103,7 @@ def test_positive_update_certified_repo_data(self):
 
         updated = self.client.get(self.build_config_url(self.certified_remote.name))
         self.assertEqual(updated.data["auth_url"], "https://auth.com")
-        self.assertEqual(updated.data["url"], "https://updated.url.com")
+        self.assertEqual(updated.data["url"], "https://updated.url.com/")
         self.assertIsNone(updated.data["requirements_file"])
 
     def test_negative_update_community_repo_data_without_requirements_file(self):
@@ -143,7 +143,7 @@ def test_positive_update_community_repo_data_with_requirements_file(self):
                     "    server: https://foobar.content.com\n"
                     "    api_key: s3cr3tk3y\n"
                 ),
-                "url": "https://galaxy.ansible.com/v3/collections",
+                "url": "https://galaxy.ansible.com/v3/collections/",
             },
             format='json'
         )