From 0df841b241d6e79b389cac598e4159a467c4f4bf Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Sat, 7 Oct 2023 23:25:29 +0200 Subject: [PATCH 01/13] add sorting and filtering tags api Issue: AAH-2761 --- galaxy_ng/app/api/ui/urls.py | 2 + galaxy_ng/app/api/ui/viewsets/__init__.py | 3 +- galaxy_ng/app/api/ui/viewsets/tags.py | 51 ++++++++++++++++++++++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/galaxy_ng/app/api/ui/urls.py b/galaxy_ng/app/api/ui/urls.py index a238a46d28..03f3e6ac02 100644 --- a/galaxy_ng/app/api/ui/urls.py +++ b/galaxy_ng/app/api/ui/urls.py @@ -25,6 +25,8 @@ router.register('distributions', viewsets.DistributionViewSet, basename='distributions') router.register('my-distributions', viewsets.MyDistributionViewSet, basename='my-distributions') +router.register('tags/collections', viewsets.CollectionsTagsViewSet, basename='collections-tags') + auth_views = [ path("login/", views.LoginView.as_view(), name="auth-login"), path("logout/", views.LogoutView.as_view(), name="auth-logout"), diff --git a/galaxy_ng/app/api/ui/viewsets/__init__.py b/galaxy_ng/app/api/ui/viewsets/__init__.py index 25c100eb12..d1328df002 100644 --- a/galaxy_ng/app/api/ui/viewsets/__init__.py +++ b/galaxy_ng/app/api/ui/viewsets/__init__.py @@ -10,7 +10,7 @@ ) from .my_namespace import MyNamespaceViewSet from .my_synclist import MySyncListViewSet -from .tags import TagsViewSet +from .tags import TagsViewSet, CollectionsTagsViewSet from .user import UserViewSet, CurrentUserViewSet from .synclist import SyncListViewSet from .root import APIRootView @@ -30,6 +30,7 @@ 'CollectionImportViewSet', 'CollectionRemoteViewSet', 'TagsViewSet', + 'CollectionsTagsViewSet', 'CurrentUserViewSet', 'UserViewSet', 'SyncListViewSet', diff --git a/galaxy_ng/app/api/ui/viewsets/tags.py b/galaxy_ng/app/api/ui/viewsets/tags.py index 6a13070250..4ec9a2b769 100644 --- a/galaxy_ng/app/api/ui/viewsets/tags.py +++ b/galaxy_ng/app/api/ui/viewsets/tags.py @@ -1,10 +1,14 @@ +from django.db.models import Count +from rest_framework import mixins +from django_filters import filters +from django_filters.rest_framework import DjangoFilterBackend, filterset + from pulp_ansible.app.models import Tag from pulp_ansible.app.serializers import TagSerializer from galaxy_ng.app.api import base as api_base -from galaxy_ng.app.access_control import access_policy - from galaxy_ng.app.api.ui import versioning +from galaxy_ng.app.access_control import access_policy class TagsViewSet(api_base.GenericViewSet): @@ -21,3 +25,46 @@ def list(self, request, *args, **kwargs): serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data) + + +class TagFilterOrdering(filters.OrderingFilter): + def filter(self, qs, value): + if value is not None and any(v in ["count", "-count"] for v in value): + order = "-" if "-count" in value else "" + + return qs.filter( + ansible_collectionversion__ansible_crossrepositorycollectionversionindex__is_highest=True # noqa: E501 + ).annotate(count=Count('ansible_collectionversion')).order_by(f"{order}count") + + return super().filter(qs, value) + + +class TagFilter(filterset.FilterSet): + sort = TagFilterOrdering( + fields=( + ("name", "name"), + ('count', 'count') + ), + ) + + class Meta: + model = Tag + fields = { + "name": ["exact", "icontains", "contains", "startswith"], + } + + +class CollectionsTagsViewSet( + api_base.GenericViewSet, + mixins.ListModelMixin +): + """ + ViewSet for collections' tags within the system. + """ + serializer_class = TagSerializer + permission_classes = [access_policy.TagsAccessPolicy] + versioning_class = versioning.UIVersioning + filter_backends = (DjangoFilterBackend,) + filterset_class = TagFilter + + queryset = Tag.objects.all() From 21edbdb0ac76aa66f9c3dac3987563524df4541a Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Sun, 8 Oct 2023 03:17:36 +0200 Subject: [PATCH 02/13] add tags/collections int test Issue: AAH-2761 --- .../tests/integration/api/test_ui_paths.py | 71 ++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/galaxy_ng/tests/integration/api/test_ui_paths.py b/galaxy_ng/tests/integration/api/test_ui_paths.py index b0756b9ec6..ce80f656fc 100644 --- a/galaxy_ng/tests/integration/api/test_ui_paths.py +++ b/galaxy_ng/tests/integration/api/test_ui_paths.py @@ -3,10 +3,13 @@ import random import pytest + +from orionutils.generator import build_collection + from ansible.galaxy.api import GalaxyError from jsonschema import validate as validate_json -from ..constants import DEFAULT_DISTROS +from ..constants import DEFAULT_DISTROS, USERNAME_PUBLISHER from ..schemas import ( schema_collection_import, schema_collection_import_detail, @@ -26,7 +29,14 @@ schema_ui_collection_summary, schema_user, ) -from ..utils import UIClient, generate_unused_namespace, get_client, wait_for_task_ui_client +from ..utils import ( + UIClient, + generate_unused_namespace, + get_client, + wait_for_task_ui_client, + wait_for_task, +) + from .rbac_actions.utils import ReusableLocalContainer REGEX_403 = r"HTTP Code: 403" @@ -767,6 +777,63 @@ def test_api_ui_v1_tags(ansible_config): # FIXME - ui tags api does not support POST? +# /api/automation-hub/_ui/v1/tags/collections/ +@pytest.mark.deployment_community +def test_api_ui_v1_tags_collections(ansible_config, upload_artifact): + + config = ansible_config("basic_user") + api_client = get_client(config) + + def build_upload_wait(tags): + artifact = build_collection( + "skeleton", + config={ + "namespace": USERNAME_PUBLISHER, + "tags": tags, + } + ) + resp = upload_artifact(config, api_client, artifact) + resp = wait_for_task(api_client, resp) + + build_upload_wait(["tools", "database", "postgresql"]) + build_upload_wait(["tools", "database", "mysql"]) + build_upload_wait(["tools", "database"]) + build_upload_wait(["tools"]) + + with UIClient(config=config) as uclient: + + # get the response + resp = uclient.get('_ui/v1/tags/collections') + assert resp.status_code == 200 + + ds = resp.json() + validate_json(instance=ds, schema=schema_objectlist) + + resp = uclient.get('_ui/v1/tags/collections?name=tools') + ds = resp.json() + assert len(ds["data"]) == 1 + + # result count should be 2 (mysql, postgresql) + resp = uclient.get('_ui/v1/tags/collections?name__icontains=sql') + ds = resp.json() + assert len(ds["data"]) == 2 + + resp = uclient.get('_ui/v1/tags/collections?name=test123') + ds = resp.json() + assert len(ds["data"]) == 0 + + # verify sort by name is correct + resp = uclient.get('_ui/v1/tags/collections?sort=name') + tags = [tag["name"] for tag in resp.json()["data"]] + assert tags == sorted(tags) + + # verify sort by -count is correct + resp = uclient.get('_ui/v1/tags/collections/?sort=-count') + data = resp.json()["data"] + assert data[0]["name"] == "tools" + assert data[1]["name"] == "database" + + # /api/automation-hub/_ui/v1/users/ @pytest.mark.deployment_standalone @pytest.mark.api_ui From 6d36539fb815a548ef446e7f3d047ec6fd9ccb51 Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Mon, 9 Oct 2023 13:00:28 +0200 Subject: [PATCH 03/13] add tags roles endpoint Issue: AAH-2761 --- galaxy_ng/app/api/ui/urls.py | 2 + galaxy_ng/app/api/ui/viewsets/__init__.py | 7 +- galaxy_ng/app/api/ui/viewsets/tags.py | 78 +++++++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/galaxy_ng/app/api/ui/urls.py b/galaxy_ng/app/api/ui/urls.py index 03f3e6ac02..fdaa1c8511 100644 --- a/galaxy_ng/app/api/ui/urls.py +++ b/galaxy_ng/app/api/ui/urls.py @@ -26,6 +26,8 @@ router.register('my-distributions', viewsets.MyDistributionViewSet, basename='my-distributions') router.register('tags/collections', viewsets.CollectionsTagsViewSet, basename='collections-tags') +router.register('tags/roles', viewsets.RolesTagsViewSet, basename='roles-tags') + auth_views = [ path("login/", views.LoginView.as_view(), name="auth-login"), diff --git a/galaxy_ng/app/api/ui/viewsets/__init__.py b/galaxy_ng/app/api/ui/viewsets/__init__.py index d1328df002..50e7deac92 100644 --- a/galaxy_ng/app/api/ui/viewsets/__init__.py +++ b/galaxy_ng/app/api/ui/viewsets/__init__.py @@ -10,7 +10,11 @@ ) from .my_namespace import MyNamespaceViewSet from .my_synclist import MySyncListViewSet -from .tags import TagsViewSet, CollectionsTagsViewSet +from .tags import ( + TagsViewSet, + CollectionsTagsViewSet, + RolesTagsViewSet +) from .user import UserViewSet, CurrentUserViewSet from .synclist import SyncListViewSet from .root import APIRootView @@ -31,6 +35,7 @@ 'CollectionRemoteViewSet', 'TagsViewSet', 'CollectionsTagsViewSet', + 'RolesTagsViewSet', 'CurrentUserViewSet', 'UserViewSet', 'SyncListViewSet', diff --git a/galaxy_ng/app/api/ui/viewsets/tags.py b/galaxy_ng/app/api/ui/viewsets/tags.py index 4ec9a2b769..4291410f80 100644 --- a/galaxy_ng/app/api/ui/viewsets/tags.py +++ b/galaxy_ng/app/api/ui/viewsets/tags.py @@ -1,4 +1,8 @@ +from itertools import chain +from gettext import gettext as _ + from django.db.models import Count +from rest_framework import serializers from rest_framework import mixins from django_filters import filters from django_filters.rest_framework import DjangoFilterBackend, filterset @@ -9,6 +13,7 @@ from galaxy_ng.app.api import base as api_base from galaxy_ng.app.api.ui import versioning from galaxy_ng.app.access_control import access_policy +from galaxy_ng.app.api.v1.models import LegacyRole class TagsViewSet(api_base.GenericViewSet): @@ -68,3 +73,76 @@ class CollectionsTagsViewSet( filterset_class = TagFilter queryset = Tag.objects.all() + + +class RolesTagFilter(filterset.FilterSet): + sort = filters.OrderingFilter( + fields=( + ("name", "name"), + ('count', 'count') + ), + ) + + class Meta: + model = LegacyRole + fields = { + "name": ["exact", "icontains", "contains", "startswith"], + } + + +class RolesTagsViewSet(api_base.GenericViewSet): + """ + ViewSet for roles' tags within the system. + """ + queryset = LegacyRole.objects.all() + permission_classes = [access_policy.TagsAccessPolicy] + versioning_class = versioning.UIVersioning + serializer_class = TagSerializer + filter_backends = (DjangoFilterBackend,) + filterset_class = RolesTagFilter + + ordering_fields = ["name", "count"] + ordering = ["name"] + filter_fields = ["exact", "icontains", "contains", "startswith"] + + def _filter_queryset(self, queryset, request): + """Custom sorting and filtering.""" + + query_params = request.query_params.copy() + sort = query_params.get("sort") + if sort: + query_params.pop("sort") + + # filtering + if value := query_params.get("name"): + queryset = list(filter(lambda x: x["name"] == value, queryset)) + elif value := query_params.get("name__contains"): + queryset = list(filter(lambda x: value in x["name"], queryset)) + elif value := query_params.get("name__icontains"): + queryset = list(filter(lambda x: value.lower() in x["name"].lower(), queryset)) + elif value := query_params.get("name__startswith"): + queryset = list(filter(lambda x: x["name"].startswith(value), queryset)) + + # sorting + if sort is not None and sort in ["name", "-name", "count", "-count"]: + reverse = True if "-" in sort else False + sort_field = sort.replace("-", "") + queryset = sorted(queryset, key=lambda x: x[sort_field], reverse=reverse) + elif sort is not None: + raise serializers.ValidationError(_(f"Invalid Sort: '{sort}'")) + + return queryset + + def list(self, request, *args, **kwargs): + + metadata_tags = LegacyRole.objects.all().values_list("full_metadata__tags", flat=True) + tag_list = list(chain(*metadata_tags)) + + tags = [dict(name=tag, count=tag_list.count(tag)) for tag in set(tag_list)] + + tags = self._filter_queryset(tags, request) + + paginator = self.pagination_class() + page = paginator.paginate_queryset(tags, request, view=self) + + return paginator.get_paginated_response(page) From d2c2a88c426dd082cfa93f1c080ea3f82ed29095 Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Mon, 9 Oct 2023 13:00:53 +0200 Subject: [PATCH 04/13] add test Issue: AAH-2761 --- .../tests/integration/api/test_ui_paths.py | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/galaxy_ng/tests/integration/api/test_ui_paths.py b/galaxy_ng/tests/integration/api/test_ui_paths.py index ce80f656fc..a2a87c7b7c 100644 --- a/galaxy_ng/tests/integration/api/test_ui_paths.py +++ b/galaxy_ng/tests/integration/api/test_ui_paths.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import random +import json import pytest @@ -36,6 +37,10 @@ wait_for_task_ui_client, wait_for_task, ) +from ..utils.legacy import ( + clean_all_roles, + wait_for_v1_task +) from .rbac_actions.utils import ReusableLocalContainer @@ -834,6 +839,124 @@ def build_upload_wait(tags): assert data[1]["name"] == "database" +# /api/automation-hub/_ui/v1/tags/roles/ +@pytest.mark.deployment_community +def test_api_ui_v1_tags_roles(ansible_config): + """Test endpoint's sorting and filtering""" + config = ansible_config("basic_user") + + admin_config = ansible_config("admin") + api_admin_client = get_client( + admin_config, + request_token=False, + require_auth=True + ) + + with UIClient(config=config) as uclient: + + # get the response + resp = uclient.get('_ui/v1/tags/roles') + assert resp.status_code == 200 + + ds = resp.json() + validate_json(instance=ds, schema=schema_objectlist) + + # clean all roles ... + clean_all_roles(ansible_config) + + # start the sync + pargs = json.dumps({"github_user": "geerlingguy", "role_name": "docker"}).encode('utf-8') + resp = api_admin_client('/api/v1/sync/', method='POST', args=pargs) + assert isinstance(resp, dict) + assert resp.get('task') is not None + wait_for_v1_task(resp=resp, api_client=api_admin_client) + + # test wrong filter param + resp = uclient.get('_ui/v1/tags/roles?wrong=filter') + resp.status_code == 200 + # assert resp.json()["errors"][0]["detail"] == "Invalid Filter: 'wrong'" + + resp = uclient.get('_ui/v1/tags/roles?name=docker') + resp.status_code == 200 + assert resp.json()["data"][0]["name"] == "docker" + + resp = uclient.get('_ui/v1/tags/roles?name=doc') + resp.status_code == 200 + assert resp.json()["meta"]["count"] == 0 + + resp = uclient.get('_ui/v1/tags/roles?name__contains=doc') + resp.status_code == 200 + assert resp.json()["data"][0]["name"] == "docker" + + resp = uclient.get('_ui/v1/tags/roles?name__contains=DOC') + resp.status_code == 200 + assert resp.json()["meta"]["count"] == 0 + + resp = uclient.get('_ui/v1/tags/roles?name__icontains=doc') + resp.status_code == 200 + assert resp.json()["data"][0]["name"] == "docker" + + resp = uclient.get('_ui/v1/tags/roles?name__icontains=DOC') + resp.status_code == 200 + assert resp.json()["data"][0]["name"] == "docker" + + resp = uclient.get('_ui/v1/tags/roles?name__startswith=doc') + resp.status_code == 200 + assert resp.json()["data"][0]["name"] == "docker" + + resp = uclient.get('_ui/v1/tags/roles?name__startswith=ker') + resp.status_code == 200 + assert resp.json()["meta"]["count"] == 0 + + # test sorting + tags = [tag for tag in uclient.get('_ui/v1/tags/roles').json()["data"]] + print("tag names: ", [(tag["name"], tag["count"]) for tag in tags]) + + resp = uclient.get('_ui/v1/tags/roles?sort=foobar') + resp.status_code == 400 + assert resp.json()["errors"][0]["detail"] == "Invalid Sort: 'foobar'" + + resp = uclient.get('_ui/v1/tags/roles?sort=name') + resp.status_code == 200 + assert sorted(tags, key=lambda x: x["name"]) == resp.json()["data"] + + # assert False + resp = uclient.get('_ui/v1/tags/roles?sort=-name') + resp.status_code == 200 + assert sorted(tags, key=lambda x: x["name"], reverse=True) == resp.json()["data"] + + resp = uclient.get('_ui/v1/tags/roles?sort=count') + resp.status_code == 200 + assert sorted(tags, key=lambda x: x["count"]) == resp.json()["data"] + + # add additional tags to test count + # tags ["docker", "system"] + pargs = json.dumps({"github_user": "6nsh", "role_name": "docker"}).encode('utf-8') + resp = api_admin_client('/api/v1/sync/', method='POST', args=pargs) + assert isinstance(resp, dict) + assert resp.get('task') is not None + wait_for_v1_task(resp=resp, api_client=api_admin_client) + + # tags ["docker"] + pargs = json.dumps({"github_user": "0x28d", "role_name": "docker_ce"}).encode('utf-8') + resp = api_admin_client('/api/v1/sync/', method='POST', args=pargs) + assert isinstance(resp, dict) + assert resp.get('task') is not None + wait_for_v1_task(resp=resp, api_client=api_admin_client) + + # test correct count sorting + tags = [tag for tag in uclient.get('_ui/v1/tags/roles').json()["data"]] + resp = uclient.get('_ui/v1/tags/roles?sort=-count') + resp.status_code == 200 + assert sorted(tags, key=lambda x: x["count"], reverse=True) == resp.json()["data"] + assert resp.json()["data"][0]["name"] == "docker" + assert resp.json()["data"][1]["name"] == "system" + + resp = uclient.get('_ui/v1/tags/roles?sort=-count&name__icontains=o') + resp.status_code == 200 + assert resp.json()["data"][0]["name"] == "docker" + + # /api/automation-hub/_ui/v1/users/ @pytest.mark.deployment_standalone @pytest.mark.api_ui From a578810a41b409967734a28ad3b50c613ebca3da Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Wed, 11 Oct 2023 18:38:12 +0200 Subject: [PATCH 05/13] add changelog Issue: AAH-2761 --- CHANGES/2761.feature | 2 ++ galaxy_ng/app/api/ui/viewsets/tags.py | 23 +++++------------------ 2 files changed, 7 insertions(+), 18 deletions(-) create mode 100644 CHANGES/2761.feature diff --git a/CHANGES/2761.feature b/CHANGES/2761.feature new file mode 100644 index 0000000000..1d71843919 --- /dev/null +++ b/CHANGES/2761.feature @@ -0,0 +1,2 @@ +Add _ui/v1/tags/collections and _ui/v1/tags/roles endpoints. +Add sorting by name and count, and enable filtering by name (exact, partial and startswith match). \ No newline at end of file diff --git a/galaxy_ng/app/api/ui/viewsets/tags.py b/galaxy_ng/app/api/ui/viewsets/tags.py index 4291410f80..3a2d275df0 100644 --- a/galaxy_ng/app/api/ui/viewsets/tags.py +++ b/galaxy_ng/app/api/ui/viewsets/tags.py @@ -75,21 +75,6 @@ class CollectionsTagsViewSet( queryset = Tag.objects.all() -class RolesTagFilter(filterset.FilterSet): - sort = filters.OrderingFilter( - fields=( - ("name", "name"), - ('count', 'count') - ), - ) - - class Meta: - model = LegacyRole - fields = { - "name": ["exact", "icontains", "contains", "startswith"], - } - - class RolesTagsViewSet(api_base.GenericViewSet): """ ViewSet for roles' tags within the system. @@ -97,16 +82,18 @@ class RolesTagsViewSet(api_base.GenericViewSet): queryset = LegacyRole.objects.all() permission_classes = [access_policy.TagsAccessPolicy] versioning_class = versioning.UIVersioning - serializer_class = TagSerializer filter_backends = (DjangoFilterBackend,) - filterset_class = RolesTagFilter ordering_fields = ["name", "count"] ordering = ["name"] filter_fields = ["exact", "icontains", "contains", "startswith"] def _filter_queryset(self, queryset, request): - """Custom sorting and filtering.""" + """ + Custom sorting and filtering, + must be performed manually since + we are overwriting the queryset with a list of tags. + """ query_params = request.query_params.copy() sort = query_params.get("sort") From 34de347ab885d6469fdaaf194ab5d3f48de9b035 Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Wed, 11 Oct 2023 19:08:06 +0200 Subject: [PATCH 06/13] remove unused test Issue: AAH-2761 --- galaxy_ng/tests/integration/api/test_ui_paths.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/galaxy_ng/tests/integration/api/test_ui_paths.py b/galaxy_ng/tests/integration/api/test_ui_paths.py index a2a87c7b7c..d2132e88f6 100644 --- a/galaxy_ng/tests/integration/api/test_ui_paths.py +++ b/galaxy_ng/tests/integration/api/test_ui_paths.py @@ -871,11 +871,6 @@ def test_api_ui_v1_tags_roles(ansible_config): assert resp.get('task') is not None wait_for_v1_task(resp=resp, api_client=api_admin_client) - # test wrong filter param - resp = uclient.get('_ui/v1/tags/roles?wrong=filter') - resp.status_code == 200 - # assert resp.json()["errors"][0]["detail"] == "Invalid Filter: 'wrong'" - resp = uclient.get('_ui/v1/tags/roles?name=docker') resp.status_code == 200 assert resp.json()["data"][0]["name"] == "docker" From e6beed621565a7e9a8b3ab9c588b3220ee43fff6 Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Wed, 11 Oct 2023 19:10:42 +0200 Subject: [PATCH 07/13] must be single line Issue: AAH-2761 --- CHANGES/2761.feature | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES/2761.feature b/CHANGES/2761.feature index 1d71843919..b12ae18928 100644 --- a/CHANGES/2761.feature +++ b/CHANGES/2761.feature @@ -1,2 +1 @@ -Add _ui/v1/tags/collections and _ui/v1/tags/roles endpoints. -Add sorting by name and count, and enable filtering by name (exact, partial and startswith match). \ No newline at end of file +Add _ui/v1/tags/collections and _ui/v1/tags/roles endpoints. Add sorting by name and count, and enable filtering by name (exact, partial and startswith match). From 98a389bc00c7777bd5ca36dcd25c673a9e0f289d Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Wed, 11 Oct 2023 19:17:50 +0200 Subject: [PATCH 08/13] fix gettext problem Issue: AAH-2761 --- galaxy_ng/app/api/ui/viewsets/tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy_ng/app/api/ui/viewsets/tags.py b/galaxy_ng/app/api/ui/viewsets/tags.py index 3a2d275df0..b6ef1718cb 100644 --- a/galaxy_ng/app/api/ui/viewsets/tags.py +++ b/galaxy_ng/app/api/ui/viewsets/tags.py @@ -116,7 +116,7 @@ def _filter_queryset(self, queryset, request): sort_field = sort.replace("-", "") queryset = sorted(queryset, key=lambda x: x[sort_field], reverse=reverse) elif sort is not None: - raise serializers.ValidationError(_(f"Invalid Sort: '{sort}'")) + raise serializers.ValidationError(_("Invalid Sort: '{}'").format(sort)) return queryset From cc078fdb7b1b920f348f2d7e5a44f5f33ff73290 Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Thu, 26 Oct 2023 02:03:28 +0200 Subject: [PATCH 09/13] add django command to populate roles tags Issue: AAH-2761 --- galaxy_ng/app/api/ui/viewsets/tags.py | 105 ++++++++---------- galaxy_ng/app/api/v1/models.py | 12 ++ galaxy_ng/app/api/v1/serializers.py | 11 +- .../management/commands/populate-role-tags.py | 24 ++++ .../0042_legacyroletag_legacyrole_tags.py | 31 ++++++ 5 files changed, 122 insertions(+), 61 deletions(-) create mode 100644 galaxy_ng/app/management/commands/populate-role-tags.py create mode 100644 galaxy_ng/app/migrations/0042_legacyroletag_legacyrole_tags.py diff --git a/galaxy_ng/app/api/ui/viewsets/tags.py b/galaxy_ng/app/api/ui/viewsets/tags.py index b6ef1718cb..476f81c9ca 100644 --- a/galaxy_ng/app/api/ui/viewsets/tags.py +++ b/galaxy_ng/app/api/ui/viewsets/tags.py @@ -1,8 +1,4 @@ -from itertools import chain -from gettext import gettext as _ - from django.db.models import Count -from rest_framework import serializers from rest_framework import mixins from django_filters import filters from django_filters.rest_framework import DjangoFilterBackend, filterset @@ -13,7 +9,8 @@ from galaxy_ng.app.api import base as api_base from galaxy_ng.app.api.ui import versioning from galaxy_ng.app.access_control import access_policy -from galaxy_ng.app.api.v1.models import LegacyRole +from galaxy_ng.app.api.v1.models import LegacyRoleTag +from galaxy_ng.app.api.v1.serializers import LegacyRoleTagSerializer class TagsViewSet(api_base.GenericViewSet): @@ -32,7 +29,7 @@ def list(self, request, *args, **kwargs): return self.get_paginated_response(serializer.data) -class TagFilterOrdering(filters.OrderingFilter): +class CollectionTagFilterOrdering(filters.OrderingFilter): def filter(self, qs, value): if value is not None and any(v in ["count", "-count"] for v in value): order = "-" if "-count" in value else "" @@ -44,8 +41,8 @@ def filter(self, qs, value): return super().filter(qs, value) -class TagFilter(filterset.FilterSet): - sort = TagFilterOrdering( +class CollectionTagFilter(filterset.FilterSet): + sort = CollectionTagFilterOrdering( fields=( ("name", "name"), ('count', 'count') @@ -70,66 +67,54 @@ class CollectionsTagsViewSet( permission_classes = [access_policy.TagsAccessPolicy] versioning_class = versioning.UIVersioning filter_backends = (DjangoFilterBackend,) - filterset_class = TagFilter + filterset_class = CollectionTagFilter queryset = Tag.objects.all() + def get_queryset(self): + qs = super().get_queryset() + return qs.annotate(count=Count("ansible_collectionversion")) -class RolesTagsViewSet(api_base.GenericViewSet): - """ - ViewSet for roles' tags within the system. - """ - queryset = LegacyRole.objects.all() - permission_classes = [access_policy.TagsAccessPolicy] - versioning_class = versioning.UIVersioning - filter_backends = (DjangoFilterBackend,) - ordering_fields = ["name", "count"] - ordering = ["name"] - filter_fields = ["exact", "icontains", "contains", "startswith"] - - def _filter_queryset(self, queryset, request): - """ - Custom sorting and filtering, - must be performed manually since - we are overwriting the queryset with a list of tags. - """ - - query_params = request.query_params.copy() - sort = query_params.get("sort") - if sort: - query_params.pop("sort") - - # filtering - if value := query_params.get("name"): - queryset = list(filter(lambda x: x["name"] == value, queryset)) - elif value := query_params.get("name__contains"): - queryset = list(filter(lambda x: value in x["name"], queryset)) - elif value := query_params.get("name__icontains"): - queryset = list(filter(lambda x: value.lower() in x["name"].lower(), queryset)) - elif value := query_params.get("name__startswith"): - queryset = list(filter(lambda x: x["name"].startswith(value), queryset)) - - # sorting - if sort is not None and sort in ["name", "-name", "count", "-count"]: - reverse = True if "-" in sort else False - sort_field = sort.replace("-", "") - queryset = sorted(queryset, key=lambda x: x[sort_field], reverse=reverse) - elif sort is not None: - raise serializers.ValidationError(_("Invalid Sort: '{}'").format(sort)) - - return queryset +class RoleTagFilterOrdering(filters.OrderingFilter): + def filter(self, qs, value): + if value is not None and any(v in ["count", "-count"] for v in value): + order = "-" if "-count" in value else "" + + return qs.annotate(count=Count('legacyrole')).order_by(f"{order}count") + + return super().filter(qs, value) - def list(self, request, *args, **kwargs): - metadata_tags = LegacyRole.objects.all().values_list("full_metadata__tags", flat=True) - tag_list = list(chain(*metadata_tags)) +class RoleTagFilter(filterset.FilterSet): + sort = RoleTagFilterOrdering( + fields=( + ("name", "name"), + ('count', 'count') + ), + ) - tags = [dict(name=tag, count=tag_list.count(tag)) for tag in set(tag_list)] + class Meta: + model = LegacyRoleTag + fields = { + "name": ["exact", "icontains", "contains", "startswith"], + } - tags = self._filter_queryset(tags, request) - paginator = self.pagination_class() - page = paginator.paginate_queryset(tags, request, view=self) +class RolesTagsViewSet( + api_base.GenericViewSet, + mixins.ListModelMixin +): + """ + ViewSet for roles' tags within the system. + """ + queryset = LegacyRoleTag.objects.all() + serializer_class = LegacyRoleTagSerializer + permission_classes = [access_policy.TagsAccessPolicy] + versioning_class = versioning.UIVersioning + filter_backends = (DjangoFilterBackend,) + filterset_class = RoleTagFilter - return paginator.get_paginated_response(page) + def get_queryset(self): + qs = super().get_queryset() + return qs.annotate(count=Count("legacyrole")) diff --git a/galaxy_ng/app/api/v1/models.py b/galaxy_ng/app/api/v1/models.py index a263b1718b..ecfe291586 100644 --- a/galaxy_ng/app/api/v1/models.py +++ b/galaxy_ng/app/api/v1/models.py @@ -115,6 +115,16 @@ def __str__(self): return self.name +class LegacyRoleTag(models.Model): + name = models.CharField(max_length=64, unique=True, editable=False) + + def __repr__(self): + return f'' + + def __str__(self): + return self.name + + class LegacyRole(models.Model): """ A legacy v1 role, which is just an index for github. @@ -154,6 +164,8 @@ class LegacyRole(models.Model): default=dict ) + tags = models.ManyToManyField(LegacyRoleTag, editable=False, related_name="legacyrole") + def __repr__(self): return f'' diff --git a/galaxy_ng/app/api/v1/serializers.py b/galaxy_ng/app/api/v1/serializers.py index 893efb3a08..b84a92ae9e 100644 --- a/galaxy_ng/app/api/v1/serializers.py +++ b/galaxy_ng/app/api/v1/serializers.py @@ -6,7 +6,7 @@ from galaxy_ng.app.models.namespace import Namespace from galaxy_ng.app.utils.rbac import get_v3_namespace_owners from galaxy_ng.app.api.v1.models import LegacyNamespace -from galaxy_ng.app.api.v1.models import LegacyRole +from galaxy_ng.app.api.v1.models import LegacyRole, LegacyRoleTag from galaxy_ng.app.api.v1.models import LegacyRoleDownloadCount from galaxy_ng.app.api.v1.utils import sort_versions @@ -602,3 +602,12 @@ class LegacyTaskDetailSerializer(serializers.Serializer): class Meta: model = None fields = ['results'] + + +class LegacyRoleTagSerializer(serializers.ModelSerializer): + + count = serializers.IntegerField(read_only=True) + + class Meta: + model = LegacyRoleTag + fields = ['name', 'count'] diff --git a/galaxy_ng/app/management/commands/populate-role-tags.py b/galaxy_ng/app/management/commands/populate-role-tags.py new file mode 100644 index 0000000000..5d7296cde8 --- /dev/null +++ b/galaxy_ng/app/management/commands/populate-role-tags.py @@ -0,0 +1,24 @@ +import django_guid +from django.core.management.base import BaseCommand + +# from galaxy_ng.app.api.v1.tasks import legacy_sync_from_upstream +from galaxy_ng.app.api.v1.models import LegacyRole, LegacyRoleTag + + +# Set logging_uid, this does not seem to get generated when task called via management command +django_guid.set_guid(django_guid.utils.generate_guid()) + + +class Command(BaseCommand): + """ + Django management command for populating role tags ('_ui/v1/tags/roles/') within the system. + This command is run nightly on galaxy.ansible.com. + """ + + help = "Populate the 'LegacyRoleTag' model with tags from LegacyRole 'full_metadata__tags'." + + def handle(self, *args, **options): + for role in LegacyRole.objects.all(): + for name in role.full_metadata["tags"]: + tag = LegacyRoleTag.objects.get_or_create(name=name) + tag[0].legacyrole.add(role) diff --git a/galaxy_ng/app/migrations/0042_legacyroletag_legacyrole_tags.py b/galaxy_ng/app/migrations/0042_legacyroletag_legacyrole_tags.py new file mode 100644 index 0000000000..a49f68f6e4 --- /dev/null +++ b/galaxy_ng/app/migrations/0042_legacyroletag_legacyrole_tags.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.6 on 2023-10-25 21:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("galaxy", "0041_alter_containerregistryremote_remote_ptr"), + ] + + operations = [ + migrations.CreateModel( + name="LegacyRoleTag", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(editable=False, max_length=64, unique=True)), + ], + ), + migrations.AddField( + model_name="legacyrole", + name="tags", + field=models.ManyToManyField( + editable=False, related_name="legacyrole", to="galaxy.legacyroletag" + ), + ), + ] From 0ff8f1f6250ce123ad55eacf26840ddddba88703 Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Thu, 26 Oct 2023 04:20:08 +0200 Subject: [PATCH 10/13] add command unit test Issue: AAH-2761 --- .../test_populate_role_tags_commands.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 galaxy_ng/tests/unit/app/management/commands/test_populate_role_tags_commands.py diff --git a/galaxy_ng/tests/unit/app/management/commands/test_populate_role_tags_commands.py b/galaxy_ng/tests/unit/app/management/commands/test_populate_role_tags_commands.py new file mode 100644 index 0000000000..be65b240d5 --- /dev/null +++ b/galaxy_ng/tests/unit/app/management/commands/test_populate_role_tags_commands.py @@ -0,0 +1,45 @@ +from django.core.management import call_command +from django.test import TestCase + +from galaxy_ng.app.api.v1.models import LegacyNamespace, LegacyRole, LegacyRoleTag + + +class TestPopulateRoleTagsCommand(TestCase): + + def _load_role(self, namespace, role, tags): + full_metadata = dict(tags=tags) + ln = LegacyNamespace.objects.get_or_create(name=namespace) + LegacyRole.objects.get_or_create(name=role, namespace=ln[0], full_metadata=full_metadata) + + def setUp(self): + super().setUp() + self._load_role("foo", "bar1", ["database", "network", "postgres"]) + self._load_role("foo", "bar2", ["database", "network"]) + + def test_populate_role_tags_command(self): + call_command('populate-role-tags') + + role_tags = LegacyRoleTag.objects.all() + tag_names = list(role_tags.values_list("name", flat=True)) + + self.assertEqual(3, role_tags.count()) + self.assertEqual(tag_names, ["database", "network", "postgres"]) + + def test_populate_twice_and_expect_same_results(self): + call_command('populate-role-tags') + role_tags_1 = LegacyRoleTag.objects.all() + self.assertEqual(3, role_tags_1.count()) + + call_command('populate-role-tags') + role_tags_2 = LegacyRoleTag.objects.all() + self.assertEqual(role_tags_1.count(), role_tags_2.count()) + + def test_populate_detected_changes(self): + call_command('populate-role-tags') + role_tags = LegacyRoleTag.objects.all() + self.assertEqual(3, role_tags.count()) + + self._load_role("foo", "bar3", ["database", "network", "mysql"]) + call_command('populate-role-tags') + role_tags = LegacyRoleTag.objects.all() + self.assertEqual(4, role_tags.count()) From fc5a28e869a7f493157880f27d28181d05f72568 Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Thu, 26 Oct 2023 05:20:39 +0200 Subject: [PATCH 11/13] refactor role tags endpoint test Issue: AAH-2761 --- .../management/commands/populate-role-tags.py | 4 +- .../tests/integration/api/test_ui_paths.py | 103 ++++++------------ 2 files changed, 35 insertions(+), 72 deletions(-) diff --git a/galaxy_ng/app/management/commands/populate-role-tags.py b/galaxy_ng/app/management/commands/populate-role-tags.py index 5d7296cde8..2f264b4732 100644 --- a/galaxy_ng/app/management/commands/populate-role-tags.py +++ b/galaxy_ng/app/management/commands/populate-role-tags.py @@ -1,3 +1,5 @@ +from gettext import gettext as _ + import django_guid from django.core.management.base import BaseCommand @@ -15,7 +17,7 @@ class Command(BaseCommand): This command is run nightly on galaxy.ansible.com. """ - help = "Populate the 'LegacyRoleTag' model with tags from LegacyRole 'full_metadata__tags'." + help = _("Populate the 'LegacyRoleTag' model with tags from LegacyRole 'full_metadata__tags'.") def handle(self, *args, **options): for role in LegacyRole.objects.all(): diff --git a/galaxy_ng/tests/integration/api/test_ui_paths.py b/galaxy_ng/tests/integration/api/test_ui_paths.py index d2132e88f6..903c39951a 100644 --- a/galaxy_ng/tests/integration/api/test_ui_paths.py +++ b/galaxy_ng/tests/integration/api/test_ui_paths.py @@ -2,11 +2,11 @@ import random import json +import subprocess import pytest from orionutils.generator import build_collection - from ansible.galaxy.api import GalaxyError from jsonschema import validate as validate_json @@ -44,6 +44,7 @@ from .rbac_actions.utils import ReusableLocalContainer + REGEX_403 = r"HTTP Code: 403" @@ -843,6 +844,21 @@ def build_upload_wait(tags): @pytest.mark.deployment_community def test_api_ui_v1_tags_roles(ansible_config): """Test endpoint's sorting and filtering""" + + def _sync_role(github_user, role_name): + pargs = json.dumps({"github_user": github_user, "role_name": role_name}).encode('utf-8') + resp = api_admin_client('/api/v1/sync/', method='POST', args=pargs) + assert isinstance(resp, dict) + assert resp.get('task') is not None + wait_for_v1_task(resp=resp, api_client=api_admin_client) + + def _populate_tags_cmd(): + proc = subprocess.run( + "django-admin populate-role-tags", + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ) + assert proc.returncode == 0 + config = ansible_config("basic_user") admin_config = ansible_config("admin") @@ -865,91 +881,36 @@ def test_api_ui_v1_tags_roles(ansible_config): clean_all_roles(ansible_config) # start the sync - pargs = json.dumps({"github_user": "geerlingguy", "role_name": "docker"}).encode('utf-8') - resp = api_admin_client('/api/v1/sync/', method='POST', args=pargs) - assert isinstance(resp, dict) - assert resp.get('task') is not None - wait_for_v1_task(resp=resp, api_client=api_admin_client) + _sync_role("geerlingguy", "docker") - resp = uclient.get('_ui/v1/tags/roles?name=docker') - resp.status_code == 200 - assert resp.json()["data"][0]["name"] == "docker" - - resp = uclient.get('_ui/v1/tags/roles?name=doc') - resp.status_code == 200 - assert resp.json()["meta"]["count"] == 0 - - resp = uclient.get('_ui/v1/tags/roles?name__contains=doc') - resp.status_code == 200 - assert resp.json()["data"][0]["name"] == "docker" - - resp = uclient.get('_ui/v1/tags/roles?name__contains=DOC') - resp.status_code == 200 - assert resp.json()["meta"]["count"] == 0 - - resp = uclient.get('_ui/v1/tags/roles?name__icontains=doc') - resp.status_code == 200 - assert resp.json()["data"][0]["name"] == "docker" - - resp = uclient.get('_ui/v1/tags/roles?name__icontains=DOC') - resp.status_code == 200 - assert resp.json()["data"][0]["name"] == "docker" - - resp = uclient.get('_ui/v1/tags/roles?name__startswith=doc') - resp.status_code == 200 - assert resp.json()["data"][0]["name"] == "docker" - - resp = uclient.get('_ui/v1/tags/roles?name__startswith=ker') + resp = uclient.get('_ui/v1/tags/roles') resp.status_code == 200 assert resp.json()["meta"]["count"] == 0 - # test sorting - tags = [tag for tag in uclient.get('_ui/v1/tags/roles').json()["data"]] - print("tag names: ", [(tag["name"], tag["count"]) for tag in tags]) - - resp = uclient.get('_ui/v1/tags/roles?sort=foobar') - resp.status_code == 400 - assert resp.json()["errors"][0]["detail"] == "Invalid Sort: 'foobar'" + # run command to populate role tags table + _populate_tags_cmd() - resp = uclient.get('_ui/v1/tags/roles?sort=name') - resp.status_code == 200 - assert sorted(tags, key=lambda x: x["name"]) == resp.json()["data"] - - # assert False - resp = uclient.get('_ui/v1/tags/roles?sort=-name') - resp.status_code == 200 - assert sorted(tags, key=lambda x: x["name"], reverse=True) == resp.json()["data"] - - resp = uclient.get('_ui/v1/tags/roles?sort=count') + resp = uclient.get('_ui/v1/tags/roles') resp.status_code == 200 - assert sorted(tags, key=lambda x: x["count"]) == resp.json()["data"] + assert resp.json()["meta"]["count"] > 0 # add additional tags to test count # tags ["docker", "system"] - pargs = json.dumps({"github_user": "6nsh", "role_name": "docker"}).encode('utf-8') - resp = api_admin_client('/api/v1/sync/', method='POST', args=pargs) - assert isinstance(resp, dict) - assert resp.get('task') is not None - wait_for_v1_task(resp=resp, api_client=api_admin_client) - + _sync_role("6nsh", "docker") # tags ["docker"] - pargs = json.dumps({"github_user": "0x28d", "role_name": "docker_ce"}).encode('utf-8') - resp = api_admin_client('/api/v1/sync/', method='POST', args=pargs) - assert isinstance(resp, dict) - assert resp.get('task') is not None - wait_for_v1_task(resp=resp, api_client=api_admin_client) + _sync_role("0x28d", "docker_ce") + _populate_tags_cmd() - # test correct count sorting - tags = [tag for tag in uclient.get('_ui/v1/tags/roles').json()["data"]] resp = uclient.get('_ui/v1/tags/roles?sort=-count') resp.status_code == 200 - assert sorted(tags, key=lambda x: x["count"], reverse=True) == resp.json()["data"] - assert resp.json()["data"][0]["name"] == "docker" - assert resp.json()["data"][1]["name"] == "system" + assert resp.json()["meta"]["count"] > 0 - resp = uclient.get('_ui/v1/tags/roles?sort=-count&name__icontains=o') - resp.status_code == 200 + # test correct count sorting + tags = [tag for tag in uclient.get('_ui/v1/tags/roles').json()["data"]] + + assert sorted(tags, key=lambda r: r["count"], reverse=True)[:2] == resp.json()["data"][:2] assert resp.json()["data"][0]["name"] == "docker" + assert resp.json()["data"][1]["name"] == "system" # /api/automation-hub/_ui/v1/users/ From 7e7f008f4696e0586c83a516581d7541144946ca Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Wed, 1 Nov 2023 12:33:50 +0100 Subject: [PATCH 12/13] rebase and add command output Issue: AAH-2761 --- .../management/commands/populate-role-tags.py | 16 +++++++++++++--- ....py => 0043_legacyroletag_legacyrole_tags.py} | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) rename galaxy_ng/app/migrations/{0042_legacyroletag_legacyrole_tags.py => 0043_legacyroletag_legacyrole_tags.py} (92%) diff --git a/galaxy_ng/app/management/commands/populate-role-tags.py b/galaxy_ng/app/management/commands/populate-role-tags.py index 2f264b4732..39ccda6183 100644 --- a/galaxy_ng/app/management/commands/populate-role-tags.py +++ b/galaxy_ng/app/management/commands/populate-role-tags.py @@ -20,7 +20,17 @@ class Command(BaseCommand): help = _("Populate the 'LegacyRoleTag' model with tags from LegacyRole 'full_metadata__tags'.") def handle(self, *args, **options): - for role in LegacyRole.objects.all(): + created_tags = [] + roles = LegacyRole.objects.all() + for role in roles: for name in role.full_metadata["tags"]: - tag = LegacyRoleTag.objects.get_or_create(name=name) - tag[0].legacyrole.add(role) + tag, created = LegacyRoleTag.objects.get_or_create(name=name) + tag.legacyrole.add(role) + + if created: + created_tags.append(tag) + + self.stdout.write( + "Successfully populated {} tags " + "from {} roles.".format(len(created_tags), len(roles)) + ) diff --git a/galaxy_ng/app/migrations/0042_legacyroletag_legacyrole_tags.py b/galaxy_ng/app/migrations/0043_legacyroletag_legacyrole_tags.py similarity index 92% rename from galaxy_ng/app/migrations/0042_legacyroletag_legacyrole_tags.py rename to galaxy_ng/app/migrations/0043_legacyroletag_legacyrole_tags.py index a49f68f6e4..5f18af92da 100644 --- a/galaxy_ng/app/migrations/0042_legacyroletag_legacyrole_tags.py +++ b/galaxy_ng/app/migrations/0043_legacyroletag_legacyrole_tags.py @@ -5,7 +5,7 @@ class Migration(migrations.Migration): dependencies = [ - ("galaxy", "0041_alter_containerregistryremote_remote_ptr"), + ("galaxy", "0042_namespace_created_namespace_updated"), ] operations = [ From 44989af72e4eea2c1447345d19852e40ebd63d77 Mon Sep 17 00:00:00 2001 From: jerabekjiri Date: Wed, 1 Nov 2023 13:18:12 +0100 Subject: [PATCH 13/13] add more info about command Issue: AAH-2761 --- galaxy_ng/app/api/ui/viewsets/tags.py | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy_ng/app/api/ui/viewsets/tags.py b/galaxy_ng/app/api/ui/viewsets/tags.py index 476f81c9ca..8021f4d82b 100644 --- a/galaxy_ng/app/api/ui/viewsets/tags.py +++ b/galaxy_ng/app/api/ui/viewsets/tags.py @@ -107,6 +107,7 @@ class RolesTagsViewSet( ): """ ViewSet for roles' tags within the system. + Tags can be populated manually by running `django-admin populate-role-tags`. """ queryset = LegacyRoleTag.objects.all() serializer_class = LegacyRoleTagSerializer